2024-11-21 11:14:24,061 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae 2024-11-21 11:14:24,077 main DEBUG Took 0.013644 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 11:14:24,078 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 11:14:24,078 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 11:14:24,079 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 11:14:24,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,097 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 11:14:24,119 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,123 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,125 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,126 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,127 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,129 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,130 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,131 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,132 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,133 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,134 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,135 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 11:14:24,136 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,136 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 11:14:24,138 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 11:14:24,140 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 11:14:24,142 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 11:14:24,142 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 11:14:24,144 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 11:14:24,144 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 11:14:24,160 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 11:14:24,166 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 11:14:24,168 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 11:14:24,170 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 11:14:24,170 main DEBUG createAppenders(={Console}) 2024-11-21 11:14:24,171 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae initialized 2024-11-21 11:14:24,171 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae 2024-11-21 11:14:24,172 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@65f8f5ae OK. 2024-11-21 11:14:24,172 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 11:14:24,173 main DEBUG OutputStream closed 2024-11-21 11:14:24,173 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 11:14:24,173 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 11:14:24,174 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@485966cc OK 2024-11-21 11:14:24,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 11:14:24,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 11:14:24,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 11:14:24,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 11:14:24,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 11:14:24,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 11:14:24,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 11:14:24,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 11:14:24,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 11:14:24,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 11:14:24,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 11:14:24,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 11:14:24,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 11:14:24,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 11:14:24,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 11:14:24,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 11:14:24,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 11:14:24,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 11:14:24,290 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 11:14:24,290 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39a2bb97) with optional ClassLoader: null 2024-11-21 11:14:24,290 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 11:14:24,291 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39a2bb97] started OK. 2024-11-21T11:14:24,309 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-21 11:14:24,313 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 11:14:24,314 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T11:14:24,687 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707 2024-11-21T11:14:24,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T11:14:24,785 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54, deleteOnExit=true 2024-11-21T11:14:24,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T11:14:24,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/test.cache.data in system properties and HBase conf 2024-11-21T11:14:24,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T11:14:24,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/hadoop.log.dir in system properties and HBase conf 2024-11-21T11:14:24,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T11:14:24,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T11:14:24,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T11:14:24,916 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T11:14:25,062 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T11:14:25,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T11:14:25,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T11:14:25,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T11:14:25,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T11:14:25,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T11:14:25,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T11:14:25,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T11:14:25,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T11:14:25,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T11:14:25,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/nfs.dump.dir in system properties and HBase conf 2024-11-21T11:14:25,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/java.io.tmpdir in system properties and HBase conf 2024-11-21T11:14:25,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T11:14:25,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T11:14:25,085 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T11:14:26,189 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T11:14:26,315 INFO [Time-limited test {}] log.Log(170): Logging initialized @3341ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T11:14:26,429 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:14:26,528 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T11:14:26,607 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T11:14:26,608 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T11:14:26,612 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T11:14:26,638 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:14:26,642 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/hadoop.log.dir/,AVAILABLE} 2024-11-21T11:14:26,643 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T11:14:26,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/java.io.tmpdir/jetty-localhost-35345-hadoop-hdfs-3_4_1-tests_jar-_-any-10426441157693481369/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T11:14:26,949 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:35345} 2024-11-21T11:14:26,949 INFO [Time-limited test {}] server.Server(415): Started @3976ms 2024-11-21T11:14:27,459 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:14:27,468 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T11:14:27,473 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T11:14:27,473 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T11:14:27,473 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T11:14:27,474 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/hadoop.log.dir/,AVAILABLE} 2024-11-21T11:14:27,475 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T11:14:27,613 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e17246{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/java.io.tmpdir/jetty-localhost-35387-hadoop-hdfs-3_4_1-tests_jar-_-any-8689237475996807443/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T11:14:27,614 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:35387} 2024-11-21T11:14:27,615 INFO [Time-limited test {}] server.Server(415): Started @4642ms 2024-11-21T11:14:27,677 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T11:14:27,920 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:14:27,927 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T11:14:27,932 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T11:14:27,932 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T11:14:27,932 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T11:14:27,937 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/hadoop.log.dir/,AVAILABLE} 2024-11-21T11:14:27,938 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T11:14:28,104 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395ff33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/java.io.tmpdir/jetty-localhost-39229-hadoop-hdfs-3_4_1-tests_jar-_-any-9159601728847886027/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T11:14:28,105 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:39229} 2024-11-21T11:14:28,105 INFO [Time-limited test {}] server.Server(415): Started @5132ms 2024-11-21T11:14:28,108 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T11:14:28,234 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T11:14:28,250 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T11:14:28,280 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T11:14:28,280 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T11:14:28,280 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T11:14:28,282 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/hadoop.log.dir/,AVAILABLE} 2024-11-21T11:14:28,283 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T11:14:28,365 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/data/data3/current/BP-1325278628-172.17.0.2-1732187665881/current, will proceed with Du for space computation calculation, 2024-11-21T11:14:28,367 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/data/data4/current/BP-1325278628-172.17.0.2-1732187665881/current, will proceed with Du for space computation calculation, 2024-11-21T11:14:28,370 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/data/data1/current/BP-1325278628-172.17.0.2-1732187665881/current, will proceed with Du for space computation calculation, 2024-11-21T11:14:28,377 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/data/data2/current/BP-1325278628-172.17.0.2-1732187665881/current, will proceed with Du for space computation calculation, 2024-11-21T11:14:28,462 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T11:14:28,469 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T11:14:28,499 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46041a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/java.io.tmpdir/jetty-localhost-46013-hadoop-hdfs-3_4_1-tests_jar-_-any-15850922724382626928/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T11:14:28,501 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:46013} 2024-11-21T11:14:28,502 INFO [Time-limited test {}] server.Server(415): Started @5529ms 2024-11-21T11:14:28,505 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T11:14:28,555 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50c68d04af4ed9c4 with lease ID 0x9ccf8cbe3fa979d8: Processing first storage report for DS-9eb6b1ed-0568-4798-bd10-36be5c339628 from datanode DatanodeRegistration(127.0.0.1:33187, datanodeUuid=502f869d-4755-4b90-bbe2-ad2016454af9, infoPort=39101, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881) 2024-11-21T11:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50c68d04af4ed9c4 with lease ID 0x9ccf8cbe3fa979d8: from storage DS-9eb6b1ed-0568-4798-bd10-36be5c339628 node DatanodeRegistration(127.0.0.1:33187, datanodeUuid=502f869d-4755-4b90-bbe2-ad2016454af9, infoPort=39101, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T11:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x530db8763f590933 with lease ID 0x9ccf8cbe3fa979d7: Processing first storage report for DS-c5c87aee-8613-492a-9420-18b36a7ae926 from datanode DatanodeRegistration(127.0.0.1:33703, datanodeUuid=10739f08-8861-4b5b-baae-bd7a7d18acb6, infoPort=32963, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881) 2024-11-21T11:14:28,557 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x530db8763f590933 with lease ID 0x9ccf8cbe3fa979d7: from storage DS-c5c87aee-8613-492a-9420-18b36a7ae926 node DatanodeRegistration(127.0.0.1:33703, datanodeUuid=10739f08-8861-4b5b-baae-bd7a7d18acb6, infoPort=32963, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T11:14:28,558 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x530db8763f590933 with lease ID 0x9ccf8cbe3fa979d7: Processing first storage report for DS-93d45973-eb15-4d6e-9eab-1b3f7042e685 from datanode DatanodeRegistration(127.0.0.1:33703, datanodeUuid=10739f08-8861-4b5b-baae-bd7a7d18acb6, infoPort=32963, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881) 2024-11-21T11:14:28,558 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x530db8763f590933 with lease ID 0x9ccf8cbe3fa979d7: from storage DS-93d45973-eb15-4d6e-9eab-1b3f7042e685 node DatanodeRegistration(127.0.0.1:33703, datanodeUuid=10739f08-8861-4b5b-baae-bd7a7d18acb6, infoPort=32963, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T11:14:28,559 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50c68d04af4ed9c4 with lease ID 0x9ccf8cbe3fa979d8: Processing first storage report for DS-ca6f8b5b-261a-4497-b1f6-ff1eca9dbe04 from datanode DatanodeRegistration(127.0.0.1:33187, datanodeUuid=502f869d-4755-4b90-bbe2-ad2016454af9, infoPort=39101, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881) 2024-11-21T11:14:28,559 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50c68d04af4ed9c4 with lease ID 0x9ccf8cbe3fa979d8: from storage DS-ca6f8b5b-261a-4497-b1f6-ff1eca9dbe04 node DatanodeRegistration(127.0.0.1:33187, datanodeUuid=502f869d-4755-4b90-bbe2-ad2016454af9, infoPort=39101, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T11:14:28,745 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/data/data5/current/BP-1325278628-172.17.0.2-1732187665881/current, will proceed with Du for space computation calculation, 2024-11-21T11:14:28,752 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/data/data6/current/BP-1325278628-172.17.0.2-1732187665881/current, will proceed with Du for space computation calculation, 2024-11-21T11:14:28,889 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T11:14:28,910 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdd3db2f2f2e98df5 with lease ID 0x9ccf8cbe3fa979d9: Processing first storage report for DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2 from datanode DatanodeRegistration(127.0.0.1:39687, datanodeUuid=92559937-3440-48c0-abcc-77d860a9b7cb, infoPort=37171, infoSecurePort=0, ipcPort=38477, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881) 2024-11-21T11:14:28,911 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd3db2f2f2e98df5 with lease ID 0x9ccf8cbe3fa979d9: from storage DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2 node DatanodeRegistration(127.0.0.1:39687, datanodeUuid=92559937-3440-48c0-abcc-77d860a9b7cb, infoPort=37171, infoSecurePort=0, ipcPort=38477, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T11:14:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdd3db2f2f2e98df5 with lease ID 0x9ccf8cbe3fa979d9: Processing first storage report for DS-f33519c3-b692-4e14-85fb-e7f361f3a76d from datanode DatanodeRegistration(127.0.0.1:39687, datanodeUuid=92559937-3440-48c0-abcc-77d860a9b7cb, infoPort=37171, infoSecurePort=0, ipcPort=38477, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881) 2024-11-21T11:14:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd3db2f2f2e98df5 with lease ID 0x9ccf8cbe3fa979d9: from storage DS-f33519c3-b692-4e14-85fb-e7f361f3a76d node DatanodeRegistration(127.0.0.1:39687, datanodeUuid=92559937-3440-48c0-abcc-77d860a9b7cb, infoPort=37171, infoSecurePort=0, ipcPort=38477, storageInfo=lv=-57;cid=testClusterID;nsid=1675839016;c=1732187665881), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T11:14:29,108 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707 2024-11-21T11:14:29,210 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/zookeeper_0, clientPort=58590, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4e660eb8-a4bd-7f42-1a2e-c346fda3a707/cluster_c99ac398-5c1d-1340-557f-0d2b79538a54/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T11:14:29,226 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58590 2024-11-21T11:14:29,249 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:29,253 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:29,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741825_1001 (size=7) 2024-11-21T11:14:29,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741825_1001 (size=7) 2024-11-21T11:14:29,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741825_1001 (size=7) 2024-11-21T11:14:29,977 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 with version=8 2024-11-21T11:14:29,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/hbase-staging 2024-11-21T11:14:30,086 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T11:14:30,418 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7b462513bfc2:0 server-side Connection retries=45 2024-11-21T11:14:30,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:30,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:30,440 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T11:14:30,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:30,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T11:14:30,641 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T11:14:30,727 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T11:14:30,740 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T11:14:30,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T11:14:30,781 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7763 (auto-detected) 2024-11-21T11:14:30,782 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T11:14:30,806 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38843 2024-11-21T11:14:30,836 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38843 connecting to ZooKeeper ensemble=127.0.0.1:58590 2024-11-21T11:14:30,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388430x0, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T11:14:30,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38843-0x1013a3c2d4c0000 connected 2024-11-21T11:14:30,954 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:30,957 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:30,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:31,007 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555, hbase.cluster.distributed=false 2024-11-21T11:14:31,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T11:14:31,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38843 2024-11-21T11:14:31,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38843 2024-11-21T11:14:31,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38843 2024-11-21T11:14:31,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38843 2024-11-21T11:14:31,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38843 2024-11-21T11:14:31,221 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b462513bfc2:0 server-side Connection retries=45 2024-11-21T11:14:31,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,224 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T11:14:31,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T11:14:31,228 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T11:14:31,230 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T11:14:31,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33361 2024-11-21T11:14:31,236 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33361 connecting to ZooKeeper ensemble=127.0.0.1:58590 2024-11-21T11:14:31,237 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,243 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333610x0, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T11:14:31,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:333610x0, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:31,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33361-0x1013a3c2d4c0001 connected 2024-11-21T11:14:31,266 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T11:14:31,284 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T11:14:31,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T11:14:31,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T11:14:31,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33361 2024-11-21T11:14:31,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33361 2024-11-21T11:14:31,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33361 2024-11-21T11:14:31,309 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33361 2024-11-21T11:14:31,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33361 2024-11-21T11:14:31,329 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b462513bfc2:0 server-side Connection retries=45 2024-11-21T11:14:31,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,330 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T11:14:31,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T11:14:31,330 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T11:14:31,331 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T11:14:31,333 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45537 2024-11-21T11:14:31,335 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45537 connecting to ZooKeeper ensemble=127.0.0.1:58590 2024-11-21T11:14:31,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,340 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:455370x0, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:31,355 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T11:14:31,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455370x0, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T11:14:31,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45537-0x1013a3c2d4c0002 connected 2024-11-21T11:14:31,362 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T11:14:31,364 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T11:14:31,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T11:14:31,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45537 2024-11-21T11:14:31,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45537 2024-11-21T11:14:31,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45537 2024-11-21T11:14:31,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45537 2024-11-21T11:14:31,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45537 2024-11-21T11:14:31,410 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b462513bfc2:0 server-side Connection retries=45 2024-11-21T11:14:31,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,411 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T11:14:31,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T11:14:31,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T11:14:31,411 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T11:14:31,412 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T11:14:31,414 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44785 2024-11-21T11:14:31,415 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44785 connecting to ZooKeeper ensemble=127.0.0.1:58590 2024-11-21T11:14:31,417 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,420 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,426 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447850x0, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:31,427 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T11:14:31,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447850x0, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T11:14:31,428 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44785-0x1013a3c2d4c0003 connected 2024-11-21T11:14:31,429 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T11:14:31,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T11:14:31,433 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T11:14:31,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44785 2024-11-21T11:14:31,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44785 2024-11-21T11:14:31,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44785 2024-11-21T11:14:31,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44785 2024-11-21T11:14:31,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44785 2024-11-21T11:14:31,478 DEBUG [M:0;7b462513bfc2:38843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7b462513bfc2:38843 2024-11-21T11:14:31,480 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7b462513bfc2,38843,1732187670157 2024-11-21T11:14:31,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,506 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7b462513bfc2,38843,1732187670157 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T11:14:31,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,543 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T11:14:31,544 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7b462513bfc2,38843,1732187670157 from backup master directory 2024-11-21T11:14:31,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7b462513bfc2,38843,1732187670157 2024-11-21T11:14:31,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T11:14:31,551 WARN [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T11:14:31,552 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7b462513bfc2,38843,1732187670157 2024-11-21T11:14:31,554 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T11:14:31,556 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T11:14:31,619 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/hbase.id] with ID: 138f8f64-bbb0-476e-9377-f6147a31aaee 2024-11-21T11:14:31,619 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/.tmp/hbase.id 2024-11-21T11:14:31,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741826_1002 (size=42) 2024-11-21T11:14:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741826_1002 (size=42) 2024-11-21T11:14:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741826_1002 (size=42) 2024-11-21T11:14:31,640 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/.tmp/hbase.id]:[hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/hbase.id] 2024-11-21T11:14:31,696 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T11:14:31,703 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T11:14:31,730 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-21T11:14:31,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:31,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741827_1003 (size=196) 2024-11-21T11:14:31,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741827_1003 (size=196) 2024-11-21T11:14:31,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741827_1003 (size=196) 2024-11-21T11:14:31,793 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:14:31,795 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T11:14:31,818 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:31,824 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:14:31,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741828_1004 (size=1189) 2024-11-21T11:14:31,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741828_1004 (size=1189) 2024-11-21T11:14:31,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741828_1004 (size=1189) 2024-11-21T11:14:31,895 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store 2024-11-21T11:14:31,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741829_1005 (size=34) 2024-11-21T11:14:31,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741829_1005 (size=34) 2024-11-21T11:14:31,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741829_1005 (size=34) 2024-11-21T11:14:31,939 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T11:14:31,944 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:31,945 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T11:14:31,946 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:31,946 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:31,948 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T11:14:31,948 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:31,948 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:31,950 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732187671945Disabling compacts and flushes for region at 1732187671945Disabling writes for close at 1732187671948 (+3 ms)Writing region close event to WAL at 1732187671948Closed at 1732187671948 2024-11-21T11:14:31,953 WARN [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/.initializing 2024-11-21T11:14:31,953 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/WALs/7b462513bfc2,38843,1732187670157 2024-11-21T11:14:31,968 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T11:14:31,989 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C38843%2C1732187670157, suffix=, logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/WALs/7b462513bfc2,38843,1732187670157, archiveDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/oldWALs, maxLogs=10 2024-11-21T11:14:32,018 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/WALs/7b462513bfc2,38843,1732187670157/7b462513bfc2%2C38843%2C1732187670157.1732187671994, exclude list is [], retry=0 2024-11-21T11:14:32,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39687,DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2,DISK] 2024-11-21T11:14:32,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33187,DS-9eb6b1ed-0568-4798-bd10-36be5c339628,DISK] 2024-11-21T11:14:32,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33703,DS-c5c87aee-8613-492a-9420-18b36a7ae926,DISK] 2024-11-21T11:14:32,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T11:14:32,102 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/WALs/7b462513bfc2,38843,1732187670157/7b462513bfc2%2C38843%2C1732187670157.1732187671994 2024-11-21T11:14:32,103 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37171:37171),(127.0.0.1/127.0.0.1:32963:32963),(127.0.0.1/127.0.0.1:39101:39101)] 2024-11-21T11:14:32,104 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:14:32,104 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:32,109 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,110 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T11:14:32,199 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:32,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,209 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T11:14:32,209 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:32,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T11:14:32,220 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:32,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,224 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T11:14:32,224 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,225 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:32,225 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,230 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,231 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,240 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,240 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,244 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T11:14:32,249 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T11:14:32,255 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:32,257 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63279481, jitterRate=-0.05706225335597992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T11:14:32,263 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732187672126Initializing all the Stores at 1732187672128 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187672129 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187672132 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187672132Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187672132Cleaning up temporary data from old regions at 1732187672241 (+109 ms)Region opened successfully at 1732187672263 (+22 ms) 2024-11-21T11:14:32,264 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T11:14:32,329 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d4c11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b462513bfc2/172.17.0.2:0 2024-11-21T11:14:32,370 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T11:14:32,385 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:32,385 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:32,389 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:32,391 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T11:14:32,396 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T11:14:32,397 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T11:14:32,436 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T11:14:32,449 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T11:14:32,451 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T11:14:32,454 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T11:14:32,456 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T11:14:32,458 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T11:14:32,460 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T11:14:32,464 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T11:14:32,466 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T11:14:32,468 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T11:14:32,470 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T11:14:32,490 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T11:14:32,492 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T11:14:32,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:32,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:32,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:32,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,502 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7b462513bfc2,38843,1732187670157, sessionid=0x1013a3c2d4c0000, setting cluster-up flag (Was=false) 2024-11-21T11:14:32,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:32,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,527 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T11:14:32,530 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b462513bfc2,38843,1732187670157 2024-11-21T11:14:32,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:32,548 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T11:14:32,550 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b462513bfc2,38843,1732187670157 2024-11-21T11:14:32,558 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T11:14:32,562 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(746): ClusterId : 138f8f64-bbb0-476e-9377-f6147a31aaee 2024-11-21T11:14:32,562 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(746): ClusterId : 138f8f64-bbb0-476e-9377-f6147a31aaee 2024-11-21T11:14:32,562 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(746): ClusterId : 138f8f64-bbb0-476e-9377-f6147a31aaee 2024-11-21T11:14:32,565 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T11:14:32,565 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T11:14:32,565 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T11:14:32,574 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T11:14:32,575 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T11:14:32,575 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T11:14:32,575 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T11:14:32,576 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T11:14:32,576 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T11:14:32,582 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T11:14:32,582 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T11:14:32,583 DEBUG [RS:1;7b462513bfc2:45537 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7b283d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b462513bfc2/172.17.0.2:0 2024-11-21T11:14:32,583 DEBUG [RS:0;7b462513bfc2:33361 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a15d28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b462513bfc2/172.17.0.2:0 2024-11-21T11:14:32,592 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T11:14:32,593 DEBUG [RS:2;7b462513bfc2:44785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f87efa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b462513bfc2/172.17.0.2:0 2024-11-21T11:14:32,608 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7b462513bfc2:33361 2024-11-21T11:14:32,612 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T11:14:32,613 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T11:14:32,613 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T11:14:32,627 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7b462513bfc2:45537 2024-11-21T11:14:32,628 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T11:14:32,628 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T11:14:32,628 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T11:14:32,631 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,38843,1732187670157 with port=33361, startcode=1732187671172 2024-11-21T11:14:32,633 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,38843,1732187670157 with port=45537, startcode=1732187671328 2024-11-21T11:14:32,639 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7b462513bfc2:44785 2024-11-21T11:14:32,640 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T11:14:32,640 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T11:14:32,640 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T11:14:32,641 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,38843,1732187670157 with port=44785, startcode=1732187671410 2024-11-21T11:14:32,648 DEBUG [RS:1;7b462513bfc2:45537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T11:14:32,648 DEBUG [RS:0;7b462513bfc2:33361 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T11:14:32,651 DEBUG [RS:2;7b462513bfc2:44785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T11:14:32,699 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:32,708 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32941, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T11:14:32,708 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49763, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T11:14:32,709 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57205, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T11:14:32,717 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,719 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T11:14:32,725 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,725 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:32,728 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T11:14:32,738 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7b462513bfc2,38843,1732187670157 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T11:14:32,747 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:14:32,747 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:14:32,748 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:14:32,748 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7b462513bfc2:0, corePoolSize=5, maxPoolSize=5 2024-11-21T11:14:32,748 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7b462513bfc2:0, corePoolSize=10, maxPoolSize=10 2024-11-21T11:14:32,748 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,748 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7b462513bfc2:0, corePoolSize=2, maxPoolSize=2 2024-11-21T11:14:32,748 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,750 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T11:14:32,750 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T11:14:32,751 WARN [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T11:14:32,751 WARN [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T11:14:32,751 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T11:14:32,751 WARN [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T11:14:32,753 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187702753 2024-11-21T11:14:32,754 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T11:14:32,755 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T11:14:32,760 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T11:14:32,760 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T11:14:32,761 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T11:14:32,761 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T11:14:32,762 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,765 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:32,765 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T11:14:32,765 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T11:14:32,766 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T11:14:32,767 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T11:14:32,772 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T11:14:32,773 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T11:14:32,774 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,775 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.large.0-1732187672774,5,FailOnTimeoutGroup] 2024-11-21T11:14:32,775 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T11:14:32,775 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.small.0-1732187672775,5,FailOnTimeoutGroup] 2024-11-21T11:14:32,775 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,776 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T11:14:32,777 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,777 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741831_1007 (size=1321) 2024-11-21T11:14:32,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741831_1007 (size=1321) 2024-11-21T11:14:32,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741831_1007 (size=1321) 2024-11-21T11:14:32,840 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T11:14:32,841 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:32,853 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,38843,1732187670157 with port=44785, startcode=1732187671410 2024-11-21T11:14:32,853 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,38843,1732187670157 with port=33361, startcode=1732187671172 2024-11-21T11:14:32,855 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b462513bfc2,38843,1732187670157 with port=45537, startcode=1732187671328 2024-11-21T11:14:32,856 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b462513bfc2,33361,1732187671172 2024-11-21T11:14:32,859 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] master.ServerManager(517): Registering regionserver=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:32,875 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b462513bfc2,44785,1732187671410 2024-11-21T11:14:32,875 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] master.ServerManager(517): Registering regionserver=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:32,880 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:32,880 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41741 2024-11-21T11:14:32,880 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T11:14:32,883 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b462513bfc2,45537,1732187671328 2024-11-21T11:14:32,884 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:32,884 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41741 2024-11-21T11:14:32,884 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T11:14:32,884 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] master.ServerManager(517): Registering regionserver=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:32,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T11:14:32,889 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:32,889 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41741 2024-11-21T11:14:32,890 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T11:14:32,892 DEBUG [RS:0;7b462513bfc2:33361 {}] zookeeper.ZKUtil(111): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b462513bfc2,33361,1732187671172 2024-11-21T11:14:32,892 WARN [RS:0;7b462513bfc2:33361 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T11:14:32,892 INFO [RS:0;7b462513bfc2:33361 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:14:32,892 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,33361,1732187671172 2024-11-21T11:14:32,893 DEBUG [RS:2;7b462513bfc2:44785 {}] zookeeper.ZKUtil(111): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b462513bfc2,44785,1732187671410 2024-11-21T11:14:32,893 WARN [RS:2;7b462513bfc2:44785 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T11:14:32,893 INFO [RS:2;7b462513bfc2:44785 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:14:32,894 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410 2024-11-21T11:14:32,895 DEBUG [RS:1;7b462513bfc2:45537 {}] zookeeper.ZKUtil(111): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b462513bfc2,45537,1732187671328 2024-11-21T11:14:32,895 WARN [RS:1;7b462513bfc2:45537 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T11:14:32,895 INFO [RS:1;7b462513bfc2:45537 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:14:32,895 DEBUG [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328 2024-11-21T11:14:32,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b462513bfc2,44785,1732187671410] 2024-11-21T11:14:32,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b462513bfc2,33361,1732187671172] 2024-11-21T11:14:32,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b462513bfc2,45537,1732187671328] 2024-11-21T11:14:32,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741832_1008 (size=32) 2024-11-21T11:14:32,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741832_1008 (size=32) 2024-11-21T11:14:32,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741832_1008 (size=32) 2024-11-21T11:14:32,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:32,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T11:14:32,928 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T11:14:32,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:32,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T11:14:32,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T11:14:32,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:32,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T11:14:32,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T11:14:32,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:32,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T11:14:32,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T11:14:32,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:32,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:32,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T11:14:32,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740 2024-11-21T11:14:32,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740 2024-11-21T11:14:32,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T11:14:32,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T11:14:32,959 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T11:14:32,960 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T11:14:32,960 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T11:14:32,962 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T11:14:32,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T11:14:32,972 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:32,973 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72397518, jitterRate=0.07880708575248718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T11:14:32,977 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T11:14:32,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732187672908Initializing all the Stores at 1732187672910 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187672910Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187672925 (+15 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187672926 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187672926Cleaning up temporary data from old regions at 1732187672958 (+32 ms)Region opened successfully at 1732187672977 (+19 ms) 2024-11-21T11:14:32,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T11:14:32,979 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T11:14:32,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T11:14:32,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T11:14:32,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T11:14:32,980 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T11:14:32,982 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T11:14:32,984 INFO [RS:2;7b462513bfc2:44785 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T11:14:32,984 INFO [RS:0;7b462513bfc2:33361 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T11:14:32,985 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T11:14:32,985 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,985 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,985 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732187672978Disabling compacts and flushes for region at 1732187672978Disabling writes for close at 1732187672979 (+1 ms)Writing region close event to WAL at 1732187672982 (+3 ms)Closed at 1732187672982 2024-11-21T11:14:32,985 INFO [RS:1;7b462513bfc2:45537 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T11:14:32,985 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,987 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T11:14:32,989 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T11:14:32,990 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:32,990 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T11:14:32,994 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T11:14:32,994 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T11:14:32,996 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,996 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,996 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,996 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,997 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,997 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,997 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0, corePoolSize=2, maxPoolSize=2 2024-11-21T11:14:32,997 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T11:14:32,997 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,997 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,997 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,998 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,998 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,998 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,998 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:14:32,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T11:14:32,998 DEBUG [RS:0;7b462513bfc2:33361 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:14:32,998 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:32,999 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,999 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,999 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,999 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:32,999 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,000 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0, corePoolSize=2, maxPoolSize=2 2024-11-21T11:14:33,000 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,000 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,000 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,000 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,000 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T11:14:33,001 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,001 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,001 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,001 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,001 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,001 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,001 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,001 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,002 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,002 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0, corePoolSize=2, maxPoolSize=2 2024-11-21T11:14:33,002 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,002 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,002 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,002 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,002 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,33361,1732187671172-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T11:14:33,002 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,002 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,002 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,002 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,002 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,003 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b462513bfc2:0, corePoolSize=1, maxPoolSize=1 2024-11-21T11:14:33,003 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:14:33,003 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:14:33,003 DEBUG [RS:2;7b462513bfc2:44785 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:14:33,003 DEBUG [RS:1;7b462513bfc2:45537 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b462513bfc2:0, corePoolSize=3, maxPoolSize=3 2024-11-21T11:14:33,014 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:33,017 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,018 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,018 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,018 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,018 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,018 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,45537,1732187671328-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T11:14:33,018 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T11:14:33,032 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T11:14:33,035 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,33361,1732187671172-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,035 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,035 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.Replication(171): 7b462513bfc2,33361,1732187671172 started 2024-11-21T11:14:33,052 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,053 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,053 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,053 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,053 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,053 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,44785,1732187671410-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T11:14:33,059 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T11:14:33,059 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,45537,1732187671328-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,059 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,060 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.Replication(171): 7b462513bfc2,45537,1732187671328 started 2024-11-21T11:14:33,062 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,063 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1482): Serving as 7b462513bfc2,33361,1732187671172, RpcServer on 7b462513bfc2/172.17.0.2:33361, sessionid=0x1013a3c2d4c0001 2024-11-21T11:14:33,064 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T11:14:33,064 DEBUG [RS:0;7b462513bfc2:33361 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b462513bfc2,33361,1732187671172 2024-11-21T11:14:33,064 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,33361,1732187671172' 2024-11-21T11:14:33,064 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T11:14:33,065 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T11:14:33,066 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T11:14:33,066 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T11:14:33,066 DEBUG [RS:0;7b462513bfc2:33361 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b462513bfc2,33361,1732187671172 2024-11-21T11:14:33,067 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,33361,1732187671172' 2024-11-21T11:14:33,067 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T11:14:33,067 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T11:14:33,068 DEBUG [RS:0;7b462513bfc2:33361 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T11:14:33,068 INFO [RS:0;7b462513bfc2:33361 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T11:14:33,069 INFO [RS:0;7b462513bfc2:33361 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T11:14:33,084 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T11:14:33,084 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,44785,1732187671410-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,084 INFO [RS:1;7b462513bfc2:45537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,084 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(1482): Serving as 7b462513bfc2,45537,1732187671328, RpcServer on 7b462513bfc2/172.17.0.2:45537, sessionid=0x1013a3c2d4c0002 2024-11-21T11:14:33,085 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T11:14:33,085 DEBUG [RS:1;7b462513bfc2:45537 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b462513bfc2,45537,1732187671328 2024-11-21T11:14:33,085 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,45537,1732187671328' 2024-11-21T11:14:33,085 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T11:14:33,085 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,086 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.Replication(171): 7b462513bfc2,44785,1732187671410 started 2024-11-21T11:14:33,088 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T11:14:33,092 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T11:14:33,092 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T11:14:33,092 DEBUG [RS:1;7b462513bfc2:45537 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b462513bfc2,45537,1732187671328 2024-11-21T11:14:33,092 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,45537,1732187671328' 2024-11-21T11:14:33,093 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T11:14:33,093 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T11:14:33,094 DEBUG [RS:1;7b462513bfc2:45537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T11:14:33,094 INFO [RS:1;7b462513bfc2:45537 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T11:14:33,094 INFO [RS:1;7b462513bfc2:45537 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T11:14:33,110 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:33,111 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1482): Serving as 7b462513bfc2,44785,1732187671410, RpcServer on 7b462513bfc2/172.17.0.2:44785, sessionid=0x1013a3c2d4c0003 2024-11-21T11:14:33,111 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T11:14:33,111 DEBUG [RS:2;7b462513bfc2:44785 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b462513bfc2,44785,1732187671410 2024-11-21T11:14:33,111 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,44785,1732187671410' 2024-11-21T11:14:33,111 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T11:14:33,112 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T11:14:33,115 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T11:14:33,116 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T11:14:33,116 DEBUG [RS:2;7b462513bfc2:44785 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b462513bfc2,44785,1732187671410 2024-11-21T11:14:33,116 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b462513bfc2,44785,1732187671410' 2024-11-21T11:14:33,116 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T11:14:33,117 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T11:14:33,120 DEBUG [RS:2;7b462513bfc2:44785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T11:14:33,120 INFO [RS:2;7b462513bfc2:44785 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T11:14:33,120 INFO [RS:2;7b462513bfc2:44785 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T11:14:33,169 WARN [7b462513bfc2:38843 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T11:14:33,176 INFO [RS:0;7b462513bfc2:33361 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T11:14:33,180 INFO [RS:0;7b462513bfc2:33361 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C33361%2C1732187671172, suffix=, logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,33361,1732187671172, archiveDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs, maxLogs=32 2024-11-21T11:14:33,195 INFO [RS:1;7b462513bfc2:45537 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T11:14:33,198 INFO [RS:1;7b462513bfc2:45537 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C45537%2C1732187671328, suffix=, logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328, archiveDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs, maxLogs=32 2024-11-21T11:14:33,201 DEBUG [RS:0;7b462513bfc2:33361 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,33361,1732187671172/7b462513bfc2%2C33361%2C1732187671172.1732187673182, exclude list is [], retry=0 2024-11-21T11:14:33,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39687,DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2,DISK] 2024-11-21T11:14:33,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33703,DS-c5c87aee-8613-492a-9420-18b36a7ae926,DISK] 2024-11-21T11:14:33,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33187,DS-9eb6b1ed-0568-4798-bd10-36be5c339628,DISK] 2024-11-21T11:14:33,216 INFO [RS:0;7b462513bfc2:33361 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,33361,1732187671172/7b462513bfc2%2C33361%2C1732187671172.1732187673182 2024-11-21T11:14:33,219 DEBUG [RS:0;7b462513bfc2:33361 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37171:37171),(127.0.0.1/127.0.0.1:32963:32963),(127.0.0.1/127.0.0.1:39101:39101)] 2024-11-21T11:14:33,221 DEBUG [RS:1;7b462513bfc2:45537 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328/7b462513bfc2%2C45537%2C1732187671328.1732187673200, exclude list is [], retry=0 2024-11-21T11:14:33,222 INFO [RS:2;7b462513bfc2:44785 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T11:14:33,227 INFO [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C44785%2C1732187671410, suffix=, logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410, archiveDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs, maxLogs=32 2024-11-21T11:14:33,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33703,DS-c5c87aee-8613-492a-9420-18b36a7ae926,DISK] 2024-11-21T11:14:33,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33187,DS-9eb6b1ed-0568-4798-bd10-36be5c339628,DISK] 2024-11-21T11:14:33,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39687,DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2,DISK] 2024-11-21T11:14:33,249 DEBUG [RS:2;7b462513bfc2:44785 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410/7b462513bfc2%2C44785%2C1732187671410.1732187673230, exclude list is [], retry=0 2024-11-21T11:14:33,249 INFO [RS:1;7b462513bfc2:45537 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328/7b462513bfc2%2C45537%2C1732187671328.1732187673200 2024-11-21T11:14:33,252 DEBUG [RS:1;7b462513bfc2:45537 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37171:37171),(127.0.0.1/127.0.0.1:39101:39101),(127.0.0.1/127.0.0.1:32963:32963)] 2024-11-21T11:14:33,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33703,DS-c5c87aee-8613-492a-9420-18b36a7ae926,DISK] 2024-11-21T11:14:33,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39687,DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2,DISK] 2024-11-21T11:14:33,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33187,DS-9eb6b1ed-0568-4798-bd10-36be5c339628,DISK] 2024-11-21T11:14:33,265 INFO [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410/7b462513bfc2%2C44785%2C1732187671410.1732187673230 2024-11-21T11:14:33,268 DEBUG [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39101:39101),(127.0.0.1/127.0.0.1:37171:37171),(127.0.0.1/127.0.0.1:32963:32963)] 2024-11-21T11:14:33,422 DEBUG [7b462513bfc2:38843 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-21T11:14:33,431 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(204): Hosts are {7b462513bfc2=0} racks are {/default-rack=0} 2024-11-21T11:14:33,439 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T11:14:33,439 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T11:14:33,439 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T11:14:33,439 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T11:14:33,440 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T11:14:33,440 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T11:14:33,440 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T11:14:33,440 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T11:14:33,440 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T11:14:33,440 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T11:14:33,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:33,457 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,45537,1732187671328, state=OPENING 2024-11-21T11:14:33,465 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T11:14:33,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:33,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:33,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:33,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:33,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,471 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:33,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:33,658 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T11:14:33,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34131, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T11:14:33,695 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T11:14:33,695 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:14:33,696 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T11:14:33,704 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C45537%2C1732187671328.meta, suffix=.meta, logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328, archiveDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs, maxLogs=32 2024-11-21T11:14:33,741 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, exclude list is [], retry=0 2024-11-21T11:14:33,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39687,DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2,DISK] 2024-11-21T11:14:33,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33703,DS-c5c87aee-8613-492a-9420-18b36a7ae926,DISK] 2024-11-21T11:14:33,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33187,DS-9eb6b1ed-0568-4798-bd10-36be5c339628,DISK] 2024-11-21T11:14:33,781 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta 2024-11-21T11:14:33,781 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37171:37171),(127.0.0.1/127.0.0.1:39101:39101),(127.0.0.1/127.0.0.1:32963:32963)] 2024-11-21T11:14:33,782 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:14:33,784 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T11:14:33,789 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T11:14:33,800 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T11:14:33,810 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T11:14:33,811 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:33,811 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T11:14:33,812 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T11:14:33,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T11:14:33,823 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T11:14:33,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:33,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T11:14:33,830 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T11:14:33,830 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:33,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T11:14:33,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T11:14:33,837 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:33,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T11:14:33,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T11:14:33,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:33,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:33,849 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T11:14:33,850 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740 2024-11-21T11:14:33,854 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740 2024-11-21T11:14:33,858 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T11:14:33,858 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T11:14:33,859 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T11:14:33,863 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T11:14:33,867 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66371291, jitterRate=-0.010990694165229797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T11:14:33,867 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T11:14:33,869 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732187673814Writing region info on filesystem at 1732187673815 (+1 ms)Initializing all the Stores at 1732187673818 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187673820 (+2 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187673820Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187673820Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187673820Cleaning up temporary data from old regions at 1732187673858 (+38 ms)Running coprocessor post-open hooks at 1732187673867 (+9 ms)Region opened successfully at 1732187673869 (+2 ms) 2024-11-21T11:14:33,882 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732187673646 2024-11-21T11:14:33,898 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T11:14:33,899 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T11:14:33,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:33,903 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,45537,1732187671328, state=OPEN 2024-11-21T11:14:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:33,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:33,910 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:33,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T11:14:33,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,45537,1732187671328 in 436 msec 2024-11-21T11:14:33,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T11:14:33,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 925 msec 2024-11-21T11:14:33,953 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:33,953 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T11:14:33,983 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:33,984 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1] 2024-11-21T11:14:34,013 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:34,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39849, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:34,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4530 sec 2024-11-21T11:14:34,053 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732187674053, completionTime=-1 2024-11-21T11:14:34,057 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-21T11:14:34,057 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:34,133 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-21T11:14:34,133 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187734133 2024-11-21T11:14:34,133 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187794133 2024-11-21T11:14:34,133 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 75 msec 2024-11-21T11:14:34,135 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:34,149 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,38843,1732187670157-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,150 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,38843,1732187670157-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,150 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,38843,1732187670157-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,153 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7b462513bfc2:38843, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,162 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,170 DEBUG [master/7b462513bfc2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T11:14:34,170 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,203 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.651sec 2024-11-21T11:14:34,205 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T11:14:34,206 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T11:14:34,208 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T11:14:34,208 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T11:14:34,208 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T11:14:34,209 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,38843,1732187670157-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T11:14:34,210 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,38843,1732187670157-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T11:14:34,215 DEBUG [master/7b462513bfc2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T11:14:34,216 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T11:14:34,216 INFO [master/7b462513bfc2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b462513bfc2,38843,1732187670157-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T11:14:34,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117ba4ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:34,279 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T11:14:34,279 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T11:14:34,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b462513bfc2,38843,-1 for getting cluster id 2024-11-21T11:14:34,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T11:14:34,301 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '138f8f64-bbb0-476e-9377-f6147a31aaee' 2024-11-21T11:14:34,304 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T11:14:34,304 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "138f8f64-bbb0-476e-9377-f6147a31aaee" 2024-11-21T11:14:34,305 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f872de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:34,305 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b462513bfc2,38843,-1] 2024-11-21T11:14:34,309 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T11:14:34,311 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:34,313 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T11:14:34,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc9d8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T11:14:34,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:34,329 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1] 2024-11-21T11:14:34,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:34,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:34,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7b462513bfc2,38843,1732187670157 2024-11-21T11:14:34,380 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T11:14:34,392 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b462513bfc2,38843,1732187670157 2024-11-21T11:14:34,396 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b548ca 2024-11-21T11:14:34,397 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T11:14:34,419 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T11:14:34,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T11:14:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:34,466 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T11:14:34,470 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:34,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-21T11:14:34,476 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T11:14:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:34,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741837_1013 (size=358) 2024-11-21T11:14:34,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741837_1013 (size=358) 2024-11-21T11:14:34,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741837_1013 (size=358) 2024-11-21T11:14:34,548 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f15a03020943182be6b4e66ea5cf19a, NAME => 'test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,565 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 78487f45e051e2d9f1275ac2ef77493d, NAME => 'test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,568 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fab4371e299cd292e600af4717485beb, NAME => 'test,bbb,1732187674423.fab4371e299cd292e600af4717485beb.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741838_1014 (size=42) 2024-11-21T11:14:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741838_1014 (size=42) 2024-11-21T11:14:34,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741838_1014 (size=42) 2024-11-21T11:14:34,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741839_1015 (size=45) 2024-11-21T11:14:34,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741839_1015 (size=45) 2024-11-21T11:14:34,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741839_1015 (size=45) 2024-11-21T11:14:34,639 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,639 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 78487f45e051e2d9f1275ac2ef77493d, disabling compactions & flushes 2024-11-21T11:14:34,639 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:34,639 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:34,639 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. after waiting 0 ms 2024-11-21T11:14:34,639 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:34,639 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:34,639 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 78487f45e051e2d9f1275ac2ef77493d: Waiting for close lock at 1732187674639Disabling compacts and flushes for region at 1732187674639Disabling writes for close at 1732187674639Writing region close event to WAL at 1732187674639Closed at 1732187674639 2024-11-21T11:14:34,640 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => be5404b662521c75e774e2749bf34474, NAME => 'test,ddd,1732187674423.be5404b662521c75e774e2749bf34474.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741840_1016 (size=45) 2024-11-21T11:14:34,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741840_1016 (size=45) 2024-11-21T11:14:34,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741840_1016 (size=45) 2024-11-21T11:14:34,663 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732187674423.fab4371e299cd292e600af4717485beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,663 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fab4371e299cd292e600af4717485beb, disabling compactions & flushes 2024-11-21T11:14:34,663 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:34,663 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:34,664 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. after waiting 0 ms 2024-11-21T11:14:34,664 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:34,664 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:34,664 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fab4371e299cd292e600af4717485beb: Waiting for close lock at 1732187674663Disabling compacts and flushes for region at 1732187674663Disabling writes for close at 1732187674664 (+1 ms)Writing region close event to WAL at 1732187674664Closed at 1732187674664 2024-11-21T11:14:34,665 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c6f9184454915671110d8238a3b71824, NAME => 'test,eee,1732187674423.c6f9184454915671110d8238a3b71824.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741841_1017 (size=45) 2024-11-21T11:14:34,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741841_1017 (size=45) 2024-11-21T11:14:34,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741841_1017 (size=45) 2024-11-21T11:14:34,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1732187674423.be5404b662521c75e774e2749bf34474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing be5404b662521c75e774e2749bf34474, disabling compactions & flushes 2024-11-21T11:14:34,704 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:34,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:34,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. after waiting 0 ms 2024-11-21T11:14:34,705 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:34,705 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:34,705 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for be5404b662521c75e774e2749bf34474: Waiting for close lock at 1732187674704Disabling compacts and flushes for region at 1732187674704Disabling writes for close at 1732187674704Writing region close event to WAL at 1732187674705 (+1 ms)Closed at 1732187674705 2024-11-21T11:14:34,705 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 76043d320d93c5a6c1b870931771d01c, NAME => 'test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741842_1018 (size=45) 2024-11-21T11:14:34,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741842_1018 (size=45) 2024-11-21T11:14:34,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741842_1018 (size=45) 2024-11-21T11:14:34,720 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732187674423.c6f9184454915671110d8238a3b71824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,720 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c6f9184454915671110d8238a3b71824, disabling compactions & flushes 2024-11-21T11:14:34,720 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:34,720 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:34,720 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732187674423.c6f9184454915671110d8238a3b71824. after waiting 0 ms 2024-11-21T11:14:34,720 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:34,720 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:34,720 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c6f9184454915671110d8238a3b71824: Waiting for close lock at 1732187674720Disabling compacts and flushes for region at 1732187674720Disabling writes for close at 1732187674720Writing region close event to WAL at 1732187674720Closed at 1732187674720 2024-11-21T11:14:34,721 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 075843f8fc377ab19ddd09be30eeaa38, NAME => 'test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741843_1019 (size=45) 2024-11-21T11:14:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741843_1019 (size=45) 2024-11-21T11:14:34,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741843_1019 (size=45) 2024-11-21T11:14:34,738 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,738 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 76043d320d93c5a6c1b870931771d01c, disabling compactions & flushes 2024-11-21T11:14:34,738 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:34,738 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:34,738 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. after waiting 0 ms 2024-11-21T11:14:34,738 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:34,738 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:34,739 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 76043d320d93c5a6c1b870931771d01c: Waiting for close lock at 1732187674738Disabling compacts and flushes for region at 1732187674738Disabling writes for close at 1732187674738Writing region close event to WAL at 1732187674738Closed at 1732187674738 2024-11-21T11:14:34,739 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 42b6f7e93058e77d80053f0460a70897, NAME => 'test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741844_1020 (size=45) 2024-11-21T11:14:34,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741844_1020 (size=45) 2024-11-21T11:14:34,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741844_1020 (size=45) 2024-11-21T11:14:34,761 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,761 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 075843f8fc377ab19ddd09be30eeaa38, disabling compactions & flushes 2024-11-21T11:14:34,761 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:34,761 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:34,761 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. after waiting 0 ms 2024-11-21T11:14:34,761 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:34,761 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:34,762 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 075843f8fc377ab19ddd09be30eeaa38: Waiting for close lock at 1732187674761Disabling compacts and flushes for region at 1732187674761Disabling writes for close at 1732187674761Writing region close event to WAL at 1732187674761Closed at 1732187674761 2024-11-21T11:14:34,762 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => feb12ac653343cadcf32bc8dff1d6846, NAME => 'test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741845_1021 (size=45) 2024-11-21T11:14:34,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741845_1021 (size=45) 2024-11-21T11:14:34,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741845_1021 (size=45) 2024-11-21T11:14:34,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 42b6f7e93058e77d80053f0460a70897, disabling compactions & flushes 2024-11-21T11:14:34,769 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:34,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:34,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. after waiting 0 ms 2024-11-21T11:14:34,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:34,769 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:34,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 42b6f7e93058e77d80053f0460a70897: Waiting for close lock at 1732187674769Disabling compacts and flushes for region at 1732187674769Disabling writes for close at 1732187674769Writing region close event to WAL at 1732187674769Closed at 1732187674769 2024-11-21T11:14:34,770 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 06bb2d53bc41f5595f048fbf33d2e7f6, NAME => 'test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741846_1022 (size=45) 2024-11-21T11:14:34,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741847_1023 (size=45) 2024-11-21T11:14:34,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741847_1023 (size=45) 2024-11-21T11:14:34,798 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741846_1022 (size=45) 2024-11-21T11:14:34,798 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing feb12ac653343cadcf32bc8dff1d6846, disabling compactions & flushes 2024-11-21T11:14:34,798 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:34,798 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:34,798 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. after waiting 0 ms 2024-11-21T11:14:34,799 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:34,799 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:34,799 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for feb12ac653343cadcf32bc8dff1d6846: Waiting for close lock at 1732187674798Disabling compacts and flushes for region at 1732187674798Disabling writes for close at 1732187674798Writing region close event to WAL at 1732187674799 (+1 ms)Closed at 1732187674799 2024-11-21T11:14:34,800 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6f3c54f1f14d06f35d9d402ce20065f9, NAME => 'test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,800 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,800 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 06bb2d53bc41f5595f048fbf33d2e7f6, disabling compactions & flushes 2024-11-21T11:14:34,800 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:34,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:34,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. after waiting 0 ms 2024-11-21T11:14:34,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:34,801 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:34,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 06bb2d53bc41f5595f048fbf33d2e7f6: Waiting for close lock at 1732187674800Disabling compacts and flushes for region at 1732187674800Disabling writes for close at 1732187674801 (+1 ms)Writing region close event to WAL at 1732187674801Closed at 1732187674801 2024-11-21T11:14:34,802 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f1d6b26cdbeafd1feed231ef1797ec16, NAME => 'test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741846_1022 (size=45) 2024-11-21T11:14:34,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741847_1023 (size=45) 2024-11-21T11:14:34,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741849_1025 (size=45) 2024-11-21T11:14:34,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741849_1025 (size=45) 2024-11-21T11:14:34,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741848_1024 (size=45) 2024-11-21T11:14:34,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741848_1024 (size=45) 2024-11-21T11:14:34,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741849_1025 (size=45) 2024-11-21T11:14:34,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741848_1024 (size=45) 2024-11-21T11:14:34,854 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,854 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f1d6b26cdbeafd1feed231ef1797ec16, disabling compactions & flushes 2024-11-21T11:14:34,854 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:34,854 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:34,854 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. after waiting 0 ms 2024-11-21T11:14:34,854 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:34,854 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:34,854 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f1d6b26cdbeafd1feed231ef1797ec16: Waiting for close lock at 1732187674854Disabling compacts and flushes for region at 1732187674854Disabling writes for close at 1732187674854Writing region close event to WAL at 1732187674854Closed at 1732187674854 2024-11-21T11:14:34,855 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ad417eadf023765c2d8ea0936fcb018c, NAME => 'test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741850_1026 (size=45) 2024-11-21T11:14:34,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741850_1026 (size=45) 2024-11-21T11:14:34,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741850_1026 (size=45) 2024-11-21T11:14:34,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ad417eadf023765c2d8ea0936fcb018c, disabling compactions & flushes 2024-11-21T11:14:34,871 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:34,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:34,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. after waiting 0 ms 2024-11-21T11:14:34,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:34,871 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:34,872 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ad417eadf023765c2d8ea0936fcb018c: Waiting for close lock at 1732187674871Disabling compacts and flushes for region at 1732187674871Disabling writes for close at 1732187674871Writing region close event to WAL at 1732187674871Closed at 1732187674871 2024-11-21T11:14:34,873 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9501094853170e6297b53847d78f71c8, NAME => 'test,nnn,1732187674423.9501094853170e6297b53847d78f71c8.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741851_1027 (size=45) 2024-11-21T11:14:34,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741851_1027 (size=45) 2024-11-21T11:14:34,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741851_1027 (size=45) 2024-11-21T11:14:34,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732187674423.9501094853170e6297b53847d78f71c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9501094853170e6297b53847d78f71c8, disabling compactions & flushes 2024-11-21T11:14:34,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:34,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:34,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. after waiting 0 ms 2024-11-21T11:14:34,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:34,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:34,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9501094853170e6297b53847d78f71c8: Waiting for close lock at 1732187674894Disabling compacts and flushes for region at 1732187674894Disabling writes for close at 1732187674894Writing region close event to WAL at 1732187674894Closed at 1732187674894 2024-11-21T11:14:34,895 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => eb15d5d5ec6aa108fb3f69b646d5757d, NAME => 'test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741852_1028 (size=45) 2024-11-21T11:14:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741852_1028 (size=45) 2024-11-21T11:14:34,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741852_1028 (size=45) 2024-11-21T11:14:34,925 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,925 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing eb15d5d5ec6aa108fb3f69b646d5757d, disabling compactions & flushes 2024-11-21T11:14:34,925 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:34,925 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:34,925 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. after waiting 0 ms 2024-11-21T11:14:34,925 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:34,925 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:34,926 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for eb15d5d5ec6aa108fb3f69b646d5757d: Waiting for close lock at 1732187674925Disabling compacts and flushes for region at 1732187674925Disabling writes for close at 1732187674925Writing region close event to WAL at 1732187674925Closed at 1732187674925 2024-11-21T11:14:34,926 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ea0e500737edd661e0f9e38b0cf0f99a, NAME => 'test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741853_1029 (size=45) 2024-11-21T11:14:34,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741853_1029 (size=45) 2024-11-21T11:14:34,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741853_1029 (size=45) 2024-11-21T11:14:34,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ea0e500737edd661e0f9e38b0cf0f99a, disabling compactions & flushes 2024-11-21T11:14:34,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:34,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:34,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. after waiting 0 ms 2024-11-21T11:14:34,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:34,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:34,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ea0e500737edd661e0f9e38b0cf0f99a: Waiting for close lock at 1732187674959Disabling compacts and flushes for region at 1732187674959Disabling writes for close at 1732187674959Writing region close event to WAL at 1732187674959Closed at 1732187674959 2024-11-21T11:14:34,960 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d2b570ae93fb4b7f17a9d44dce21589f, NAME => 'test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:34,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741854_1030 (size=45) 2024-11-21T11:14:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741854_1030 (size=45) 2024-11-21T11:14:34,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741854_1030 (size=45) 2024-11-21T11:14:34,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:34,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d2b570ae93fb4b7f17a9d44dce21589f, disabling compactions & flushes 2024-11-21T11:14:34,990 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:34,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:34,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. after waiting 0 ms 2024-11-21T11:14:34,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:34,990 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:34,990 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d2b570ae93fb4b7f17a9d44dce21589f: Waiting for close lock at 1732187674990Disabling compacts and flushes for region at 1732187674990Disabling writes for close at 1732187674990Writing region close event to WAL at 1732187674990Closed at 1732187674990 2024-11-21T11:14:34,991 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => aa29bf2ce1b3154b66b07cc896f4188a, NAME => 'test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741855_1031 (size=45) 2024-11-21T11:14:35,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741855_1031 (size=45) 2024-11-21T11:14:35,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741855_1031 (size=45) 2024-11-21T11:14:35,006 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,006 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing aa29bf2ce1b3154b66b07cc896f4188a, disabling compactions & flushes 2024-11-21T11:14:35,007 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:35,007 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:35,007 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. after waiting 0 ms 2024-11-21T11:14:35,007 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:35,007 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:35,007 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for aa29bf2ce1b3154b66b07cc896f4188a: Waiting for close lock at 1732187675006Disabling compacts and flushes for region at 1732187675006Disabling writes for close at 1732187675007 (+1 ms)Writing region close event to WAL at 1732187675007Closed at 1732187675007 2024-11-21T11:14:35,008 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8f738404802b7de06817afff11fed6aa, NAME => 'test,sss,1732187674423.8f738404802b7de06817afff11fed6aa.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,028 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3f15a03020943182be6b4e66ea5cf19a, disabling compactions & flushes 2024-11-21T11:14:35,029 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:35,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:35,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. after waiting 0 ms 2024-11-21T11:14:35,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:35,029 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:35,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f15a03020943182be6b4e66ea5cf19a: Waiting for close lock at 1732187675029Disabling compacts and flushes for region at 1732187675029Disabling writes for close at 1732187675029Writing region close event to WAL at 1732187675029Closed at 1732187675029 2024-11-21T11:14:35,030 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c3ba6d97819a96bf36b5d4ae2cc3573, NAME => 'test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741856_1032 (size=45) 2024-11-21T11:14:35,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741856_1032 (size=45) 2024-11-21T11:14:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741856_1032 (size=45) 2024-11-21T11:14:35,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1732187674423.8f738404802b7de06817afff11fed6aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8f738404802b7de06817afff11fed6aa, disabling compactions & flushes 2024-11-21T11:14:35,052 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:35,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:35,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. after waiting 0 ms 2024-11-21T11:14:35,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:35,052 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:35,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8f738404802b7de06817afff11fed6aa: Waiting for close lock at 1732187675052Disabling compacts and flushes for region at 1732187675052Disabling writes for close at 1732187675052Writing region close event to WAL at 1732187675052Closed at 1732187675052 2024-11-21T11:14:35,053 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 46a04157c564adf02211263396700675, NAME => 'test,uuu,1732187674423.46a04157c564adf02211263396700675.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741857_1033 (size=45) 2024-11-21T11:14:35,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741857_1033 (size=45) 2024-11-21T11:14:35,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741857_1033 (size=45) 2024-11-21T11:14:35,081 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,081 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9c3ba6d97819a96bf36b5d4ae2cc3573, disabling compactions & flushes 2024-11-21T11:14:35,081 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:35,081 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:35,081 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. after waiting 0 ms 2024-11-21T11:14:35,081 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:35,081 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:35,081 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c3ba6d97819a96bf36b5d4ae2cc3573: Waiting for close lock at 1732187675081Disabling compacts and flushes for region at 1732187675081Disabling writes for close at 1732187675081Writing region close event to WAL at 1732187675081Closed at 1732187675081 2024-11-21T11:14:35,082 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5ad652628fe53a6af579af31b1d64bdf, NAME => 'test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741858_1034 (size=45) 2024-11-21T11:14:35,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741858_1034 (size=45) 2024-11-21T11:14:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741858_1034 (size=45) 2024-11-21T11:14:35,094 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1732187674423.46a04157c564adf02211263396700675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 46a04157c564adf02211263396700675, disabling compactions & flushes 2024-11-21T11:14:35,101 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:35,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:35,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732187674423.46a04157c564adf02211263396700675. after waiting 0 ms 2024-11-21T11:14:35,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:35,101 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:35,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 46a04157c564adf02211263396700675: Waiting for close lock at 1732187675101Disabling compacts and flushes for region at 1732187675101Disabling writes for close at 1732187675101Writing region close event to WAL at 1732187675101Closed at 1732187675101 2024-11-21T11:14:35,102 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 05dba68ed06ee6ea199d777ec417cb17, NAME => 'test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:35,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741859_1035 (size=45) 2024-11-21T11:14:35,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741859_1035 (size=45) 2024-11-21T11:14:35,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741859_1035 (size=45) 2024-11-21T11:14:35,123 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,124 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5ad652628fe53a6af579af31b1d64bdf, disabling compactions & flushes 2024-11-21T11:14:35,124 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:35,124 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:35,124 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. after waiting 0 ms 2024-11-21T11:14:35,124 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:35,124 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:35,124 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5ad652628fe53a6af579af31b1d64bdf: Waiting for close lock at 1732187675124Disabling compacts and flushes for region at 1732187675124Disabling writes for close at 1732187675124Writing region close event to WAL at 1732187675124Closed at 1732187675124 2024-11-21T11:14:35,125 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 993840b9e61b5baf1c5a77b64bde7e9d, NAME => 'test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741861_1037 (size=45) 2024-11-21T11:14:35,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741861_1037 (size=45) 2024-11-21T11:14:35,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741861_1037 (size=45) 2024-11-21T11:14:35,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741860_1036 (size=45) 2024-11-21T11:14:35,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741860_1036 (size=45) 2024-11-21T11:14:35,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741860_1036 (size=45) 2024-11-21T11:14:35,160 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,160 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 05dba68ed06ee6ea199d777ec417cb17, disabling compactions & flushes 2024-11-21T11:14:35,160 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:35,160 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:35,160 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. after waiting 0 ms 2024-11-21T11:14:35,160 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:35,160 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:35,160 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 05dba68ed06ee6ea199d777ec417cb17: Waiting for close lock at 1732187675160Disabling compacts and flushes for region at 1732187675160Disabling writes for close at 1732187675160Writing region close event to WAL at 1732187675160Closed at 1732187675160 2024-11-21T11:14:35,161 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f52493bc04c75885babe0eeaf6faf9b0, NAME => 'test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741862_1038 (size=45) 2024-11-21T11:14:35,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741862_1038 (size=45) 2024-11-21T11:14:35,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741862_1038 (size=45) 2024-11-21T11:14:35,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f52493bc04c75885babe0eeaf6faf9b0, disabling compactions & flushes 2024-11-21T11:14:35,189 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:35,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:35,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. after waiting 0 ms 2024-11-21T11:14:35,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:35,189 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:35,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f52493bc04c75885babe0eeaf6faf9b0: Waiting for close lock at 1732187675189Disabling compacts and flushes for region at 1732187675189Disabling writes for close at 1732187675189Writing region close event to WAL at 1732187675189Closed at 1732187675189 2024-11-21T11:14:35,190 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 325cd18a1a1b735062fa7f51132edef4, NAME => 'test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555 2024-11-21T11:14:35,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741863_1039 (size=42) 2024-11-21T11:14:35,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741863_1039 (size=42) 2024-11-21T11:14:35,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741863_1039 (size=42) 2024-11-21T11:14:35,226 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,226 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 325cd18a1a1b735062fa7f51132edef4, disabling compactions & flushes 2024-11-21T11:14:35,226 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:35,226 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:35,226 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. after waiting 0 ms 2024-11-21T11:14:35,226 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:35,226 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:35,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 325cd18a1a1b735062fa7f51132edef4: Waiting for close lock at 1732187675226Disabling compacts and flushes for region at 1732187675226Disabling writes for close at 1732187675226Writing region close event to WAL at 1732187675226Closed at 1732187675226 2024-11-21T11:14:35,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6f3c54f1f14d06f35d9d402ce20065f9, disabling compactions & flushes 2024-11-21T11:14:35,250 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:35,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:35,251 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. after waiting 0 ms 2024-11-21T11:14:35,251 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:35,251 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:35,251 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6f3c54f1f14d06f35d9d402ce20065f9: Waiting for close lock at 1732187675250Disabling compacts and flushes for region at 1732187675250Disabling writes for close at 1732187675251 (+1 ms)Writing region close event to WAL at 1732187675251Closed at 1732187675251 2024-11-21T11:14:35,555 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:35,555 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 993840b9e61b5baf1c5a77b64bde7e9d, disabling compactions & flushes 2024-11-21T11:14:35,555 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:35,555 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:35,555 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. after waiting 0 ms 2024-11-21T11:14:35,555 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:35,555 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:35,555 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 993840b9e61b5baf1c5a77b64bde7e9d: Waiting for close lock at 1732187675555Disabling compacts and flushes for region at 1732187675555Disabling writes for close at 1732187675555Writing region close event to WAL at 1732187675555Closed at 1732187675555 2024-11-21T11:14:35,558 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T11:14:35,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732187674423.fab4371e299cd292e600af4717485beb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732187674423.be5404b662521c75e774e2749bf34474.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732187674423.c6f9184454915671110d8238a3b71824.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732187674423.9501094853170e6297b53847d78f71c8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732187674423.8f738404802b7de06817afff11fed6aa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732187674423.46a04157c564adf02211263396700675.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,569 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732187675558"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732187675558"}]},"ts":"1732187675558"} 2024-11-21T11:14:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:35,622 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-21T11:14:35,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T11:14:35,627 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187675624"}]},"ts":"1732187675624"} 2024-11-21T11:14:35,632 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-21T11:14:35,633 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {7b462513bfc2=0} racks are {/default-rack=0} 2024-11-21T11:14:35,654 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T11:14:35,654 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T11:14:35,654 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T11:14:35,654 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T11:14:35,655 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T11:14:35,655 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T11:14:35,655 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T11:14:35,655 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T11:14:35,655 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T11:14:35,655 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T11:14:35,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3f15a03020943182be6b4e66ea5cf19a, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fab4371e299cd292e600af4717485beb, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=be5404b662521c75e774e2749bf34474, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76043d320d93c5a6c1b870931771d01c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=075843f8fc377ab19ddd09be30eeaa38, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=feb12ac653343cadcf32bc8dff1d6846, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06bb2d53bc41f5595f048fbf33d2e7f6, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad417eadf023765c2d8ea0936fcb018c, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9501094853170e6297b53847d78f71c8, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ea0e500737edd661e0f9e38b0cf0f99a, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b570ae93fb4b7f17a9d44dce21589f, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8f738404802b7de06817afff11fed6aa, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46a04157c564adf02211263396700675, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5ad652628fe53a6af579af31b1d64bdf, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=993840b9e61b5baf1c5a77b64bde7e9d, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f52493bc04c75885babe0eeaf6faf9b0, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=325cd18a1a1b735062fa7f51132edef4, ASSIGN}] 2024-11-21T11:14:35,669 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=325cd18a1a1b735062fa7f51132edef4, ASSIGN 2024-11-21T11:14:35,671 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=325cd18a1a1b735062fa7f51132edef4, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f52493bc04c75885babe0eeaf6faf9b0, ASSIGN 2024-11-21T11:14:35,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f52493bc04c75885babe0eeaf6faf9b0, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,678 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=993840b9e61b5baf1c5a77b64bde7e9d, ASSIGN 2024-11-21T11:14:35,680 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=993840b9e61b5baf1c5a77b64bde7e9d, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,684 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:35,685 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,687 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5ad652628fe53a6af579af31b1d64bdf, ASSIGN 2024-11-21T11:14:35,696 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5ad652628fe53a6af579af31b1d64bdf, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,698 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46a04157c564adf02211263396700675, ASSIGN 2024-11-21T11:14:35,700 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=46a04157c564adf02211263396700675, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,702 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:35,703 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,705 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8f738404802b7de06817afff11fed6aa, ASSIGN 2024-11-21T11:14:35,707 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8f738404802b7de06817afff11fed6aa, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,709 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:35,711 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,712 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b570ae93fb4b7f17a9d44dce21589f, ASSIGN 2024-11-21T11:14:35,717 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d2b570ae93fb4b7f17a9d44dce21589f, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,721 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ea0e500737edd661e0f9e38b0cf0f99a, ASSIGN 2024-11-21T11:14:35,724 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ea0e500737edd661e0f9e38b0cf0f99a, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,725 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, ASSIGN 2024-11-21T11:14:35,727 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,728 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9501094853170e6297b53847d78f71c8, ASSIGN 2024-11-21T11:14:35,730 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9501094853170e6297b53847d78f71c8, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,733 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad417eadf023765c2d8ea0936fcb018c, ASSIGN 2024-11-21T11:14:35,734 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad417eadf023765c2d8ea0936fcb018c, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,736 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:35,738 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,740 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:35,741 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,743 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06bb2d53bc41f5595f048fbf33d2e7f6, ASSIGN 2024-11-21T11:14:35,745 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=06bb2d53bc41f5595f048fbf33d2e7f6, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,746 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=feb12ac653343cadcf32bc8dff1d6846, ASSIGN 2024-11-21T11:14:35,747 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=feb12ac653343cadcf32bc8dff1d6846, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,749 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:35,751 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,754 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=075843f8fc377ab19ddd09be30eeaa38, ASSIGN 2024-11-21T11:14:35,755 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=075843f8fc377ab19ddd09be30eeaa38, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,757 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76043d320d93c5a6c1b870931771d01c, ASSIGN 2024-11-21T11:14:35,758 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=76043d320d93c5a6c1b870931771d01c, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:35,761 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,763 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=be5404b662521c75e774e2749bf34474, ASSIGN 2024-11-21T11:14:35,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=be5404b662521c75e774e2749bf34474, ASSIGN; state=OFFLINE, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:35,765 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:35,766 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN; state=OFFLINE, location=7b462513bfc2,45537,1732187671328; forceNewPlan=false, retain=false 2024-11-21T11:14:35,768 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fab4371e299cd292e600af4717485beb, ASSIGN 2024-11-21T11:14:35,772 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fab4371e299cd292e600af4717485beb, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3f15a03020943182be6b4e66ea5cf19a, ASSIGN 2024-11-21T11:14:35,775 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3f15a03020943182be6b4e66ea5cf19a, ASSIGN; state=OFFLINE, location=7b462513bfc2,44785,1732187671410; forceNewPlan=false, retain=false 2024-11-21T11:14:35,825 INFO [7b462513bfc2:38843 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-21T11:14:35,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,829 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN because future has completed 2024-11-21T11:14:35,830 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,831 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,833 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c6f9184454915671110d8238a3b71824, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,833 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:35,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN because future has completed 2024-11-21T11:14:35,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN because future has completed 2024-11-21T11:14:35,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, ASSIGN because future has completed 2024-11-21T11:14:35,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN because future has completed 2024-11-21T11:14:35,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN because future has completed 2024-11-21T11:14:35,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN because future has completed 2024-11-21T11:14:35,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN because future has completed 2024-11-21T11:14:35,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN because future has completed 2024-11-21T11:14:35,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:35,911 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,912 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=46a04157c564adf02211263396700675, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ad417eadf023765c2d8ea0936fcb018c, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=feb12ac653343cadcf32bc8dff1d6846, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f52493bc04c75885babe0eeaf6faf9b0, ASSIGN because future has completed 2024-11-21T11:14:35,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f52493bc04c75885babe0eeaf6faf9b0, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=993840b9e61b5baf1c5a77b64bde7e9d, ASSIGN because future has completed 2024-11-21T11:14:35,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=46a04157c564adf02211263396700675, ASSIGN because future has completed 2024-11-21T11:14:35,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ea0e500737edd661e0f9e38b0cf0f99a, ASSIGN because future has completed 2024-11-21T11:14:35,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad417eadf023765c2d8ea0936fcb018c, ASSIGN because future has completed 2024-11-21T11:14:35,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad417eadf023765c2d8ea0936fcb018c, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=feb12ac653343cadcf32bc8dff1d6846, ASSIGN because future has completed 2024-11-21T11:14:35,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure feb12ac653343cadcf32bc8dff1d6846, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea0e500737edd661e0f9e38b0cf0f99a, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46a04157c564adf02211263396700675, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 993840b9e61b5baf1c5a77b64bde7e9d, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=76043d320d93c5a6c1b870931771d01c, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=fab4371e299cd292e600af4717485beb, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3f15a03020943182be6b4e66ea5cf19a, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:35,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=325cd18a1a1b735062fa7f51132edef4, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:35,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5ad652628fe53a6af579af31b1d64bdf, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:35,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8f738404802b7de06817afff11fed6aa, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:35,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=76043d320d93c5a6c1b870931771d01c, ASSIGN because future has completed 2024-11-21T11:14:35,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76043d320d93c5a6c1b870931771d01c, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:35,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fab4371e299cd292e600af4717485beb, ASSIGN because future has completed 2024-11-21T11:14:35,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3f15a03020943182be6b4e66ea5cf19a, ASSIGN because future has completed 2024-11-21T11:14:35,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=325cd18a1a1b735062fa7f51132edef4, ASSIGN because future has completed 2024-11-21T11:14:35,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5ad652628fe53a6af579af31b1d64bdf, ASSIGN because future has completed 2024-11-21T11:14:35,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8f738404802b7de06817afff11fed6aa, ASSIGN because future has completed 2024-11-21T11:14:35,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f738404802b7de06817afff11fed6aa, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ad652628fe53a6af579af31b1d64bdf, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 325cd18a1a1b735062fa7f51132edef4, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f15a03020943182be6b4e66ea5cf19a, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:36,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure fab4371e299cd292e600af4717485beb, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:36,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=9501094853170e6297b53847d78f71c8, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,033 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:36,034 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => eb15d5d5ec6aa108fb3f69b646d5757d, NAME => 'test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-21T11:14:36,035 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,035 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=075843f8fc377ab19ddd09be30eeaa38, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,035 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,035 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,036 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=be5404b662521c75e774e2749bf34474, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d2b570ae93fb4b7f17a9d44dce21589f, ASSIGN because future has completed 2024-11-21T11:14:36,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b570ae93fb4b7f17a9d44dce21589f, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9501094853170e6297b53847d78f71c8, ASSIGN because future has completed 2024-11-21T11:14:36,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=06bb2d53bc41f5595f048fbf33d2e7f6, ASSIGN because future has completed 2024-11-21T11:14:36,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06bb2d53bc41f5595f048fbf33d2e7f6, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=075843f8fc377ab19ddd09be30eeaa38, ASSIGN because future has completed 2024-11-21T11:14:36,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=be5404b662521c75e774e2749bf34474, ASSIGN because future has completed 2024-11-21T11:14:36,050 INFO [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,052 INFO [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb15d5d5ec6aa108fb3f69b646d5757d columnFamilyName family 2024-11-21T11:14:36,053 DEBUG [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure be5404b662521c75e774e2749bf34474, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,054 INFO [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] regionserver.HStore(327): Store=eb15d5d5ec6aa108fb3f69b646d5757d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,054 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,055 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,056 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,057 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,057 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 075843f8fc377ab19ddd09be30eeaa38, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,062 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9501094853170e6297b53847d78f71c8, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:36,073 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,074 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened eb15d5d5ec6aa108fb3f69b646d5757d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60436397, jitterRate=-0.09942750632762909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,074 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:36,075 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for eb15d5d5ec6aa108fb3f69b646d5757d: Running coprocessor pre-open hook at 1732187676036Writing region info on filesystem at 1732187676036Initializing all the Stores at 1732187676038 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676038Cleaning up temporary data from old regions at 1732187676057 (+19 ms)Running coprocessor post-open hooks at 1732187676074 (+17 ms)Region opened successfully at 1732187676075 (+1 ms) 2024-11-21T11:14:36,077 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d., pid=32, masterSystemTime=1732187676012 2024-11-21T11:14:36,078 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T11:14:36,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53813, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T11:14:36,084 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:36,084 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:36,084 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:36,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,084 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => f1d6b26cdbeafd1feed231ef1797ec16, NAME => 'test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-21T11:14:36,085 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,085 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,085 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,085 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=19 2024-11-21T11:14:36,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,45537,1732187671328 in 244 msec 2024-11-21T11:14:36,100 INFO [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,100 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:36,101 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => f52493bc04c75885babe0eeaf6faf9b0, NAME => 'test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-21T11:14:36,101 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,101 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,101 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,101 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, ASSIGN in 442 msec 2024-11-21T11:14:36,102 INFO [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1d6b26cdbeafd1feed231ef1797ec16 columnFamilyName family 2024-11-21T11:14:36,102 DEBUG [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,103 INFO [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] regionserver.HStore(327): Store=f1d6b26cdbeafd1feed231ef1797ec16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,103 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,103 INFO [StoreOpener-f52493bc04c75885babe0eeaf6faf9b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,104 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,106 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,106 INFO [StoreOpener-f52493bc04c75885babe0eeaf6faf9b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f52493bc04c75885babe0eeaf6faf9b0 columnFamilyName family 2024-11-21T11:14:36,107 DEBUG [StoreOpener-f52493bc04c75885babe0eeaf6faf9b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,113 INFO [StoreOpener-f52493bc04c75885babe0eeaf6faf9b0-1 {}] regionserver.HStore(327): Store=f52493bc04c75885babe0eeaf6faf9b0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,114 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,114 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,118 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,120 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,123 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,124 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,125 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,125 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,129 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,137 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f1d6b26cdbeafd1feed231ef1797ec16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,138 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened f1d6b26cdbeafd1feed231ef1797ec16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64191198, jitterRate=-0.043476611375808716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,138 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:36,138 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for f1d6b26cdbeafd1feed231ef1797ec16: Running coprocessor pre-open hook at 1732187676085Writing region info on filesystem at 1732187676085Initializing all the Stores at 1732187676096 (+11 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676097 (+1 ms)Cleaning up temporary data from old regions at 1732187676114 (+17 ms)Running coprocessor post-open hooks at 1732187676138 (+24 ms)Region opened successfully at 1732187676138 2024-11-21T11:14:36,140 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16., pid=37, masterSystemTime=1732187676012 2024-11-21T11:14:36,145 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:36,145 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:36,145 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:36,145 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c3ba6d97819a96bf36b5d4ae2cc3573, NAME => 'test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-21T11:14:36,146 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,146 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,146 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,146 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,150 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T11:14:36,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,153 INFO [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,158 INFO [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c3ba6d97819a96bf36b5d4ae2cc3573 columnFamilyName family 2024-11-21T11:14:36,158 DEBUG [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=16 2024-11-21T11:14:36,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,45537,1732187671328 in 270 msec 2024-11-21T11:14:36,159 INFO [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] regionserver.HStore(327): Store=9c3ba6d97819a96bf36b5d4ae2cc3573/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,159 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,160 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,161 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN in 502 msec 2024-11-21T11:14:36,162 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,163 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,164 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f52493bc04c75885babe0eeaf6faf9b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,165 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened f52493bc04c75885babe0eeaf6faf9b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72440934, jitterRate=0.07945403456687927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,165 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:36,166 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for f52493bc04c75885babe0eeaf6faf9b0: Running coprocessor pre-open hook at 1732187676102Writing region info on filesystem at 1732187676102Initializing all the Stores at 1732187676103 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676103Cleaning up temporary data from old regions at 1732187676125 (+22 ms)Running coprocessor post-open hooks at 1732187676165 (+40 ms)Region opened successfully at 1732187676165 2024-11-21T11:14:36,167 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0., pid=40, masterSystemTime=1732187676077 2024-11-21T11:14:36,167 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54867, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T11:14:36,173 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:36,173 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:36,173 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9c3ba6d97819a96bf36b5d4ae2cc3573/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,173 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,173 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:36,174 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => fab4371e299cd292e600af4717485beb, NAME => 'test,bbb,1732187674423.fab4371e299cd292e600af4717485beb.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-21T11:14:36,174 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,174 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,bbb,1732187674423.fab4371e299cd292e600af4717485beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,174 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,175 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,175 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 9c3ba6d97819a96bf36b5d4ae2cc3573; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73173625, jitterRate=0.0903719812631607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,175 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:36,175 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 9c3ba6d97819a96bf36b5d4ae2cc3573: Running coprocessor pre-open hook at 1732187676146Writing region info on filesystem at 1732187676146Initializing all the Stores at 1732187676150 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676150Cleaning up temporary data from old regions at 1732187676163 (+13 ms)Running coprocessor post-open hooks at 1732187676175 (+12 ms)Region opened successfully at 1732187676175 2024-11-21T11:14:36,177 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573., pid=39, masterSystemTime=1732187676012 2024-11-21T11:14:36,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f52493bc04c75885babe0eeaf6faf9b0, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,179 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:36,180 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => be5404b662521c75e774e2749bf34474, NAME => 'test,ddd,1732187674423.be5404b662521c75e774e2749bf34474.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-21T11:14:36,180 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,181 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ddd,1732187674423.be5404b662521c75e774e2749bf34474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,181 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,181 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,182 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:36,183 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:36,183 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:36,184 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => c6f9184454915671110d8238a3b71824, NAME => 'test,eee,1732187674423.c6f9184454915671110d8238a3b71824.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-21T11:14:36,184 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,184 INFO [StoreOpener-fab4371e299cd292e600af4717485beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,185 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,eee,1732187674423.c6f9184454915671110d8238a3b71824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=29 2024-11-21T11:14:36,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f52493bc04c75885babe0eeaf6faf9b0, server=7b462513bfc2,44785,1732187671410 in 260 msec 2024-11-21T11:14:36,185 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,185 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f52493bc04c75885babe0eeaf6faf9b0, ASSIGN in 528 msec 2024-11-21T11:14:36,188 INFO [StoreOpener-fab4371e299cd292e600af4717485beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fab4371e299cd292e600af4717485beb columnFamilyName family 2024-11-21T11:14:36,189 DEBUG [StoreOpener-fab4371e299cd292e600af4717485beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,190 INFO [StoreOpener-fab4371e299cd292e600af4717485beb-1 {}] regionserver.HStore(327): Store=fab4371e299cd292e600af4717485beb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,190 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,191 INFO [StoreOpener-be5404b662521c75e774e2749bf34474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,191 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,192 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,193 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,193 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,197 INFO [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,198 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-21T11:14:36,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,45537,1732187671328 in 292 msec 2024-11-21T11:14:36,200 INFO [StoreOpener-be5404b662521c75e774e2749bf34474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be5404b662521c75e774e2749bf34474 columnFamilyName family 2024-11-21T11:14:36,200 DEBUG [StoreOpener-be5404b662521c75e774e2749bf34474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,201 INFO [StoreOpener-be5404b662521c75e774e2749bf34474-1 {}] regionserver.HStore(327): Store=be5404b662521c75e774e2749bf34474/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,201 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN in 542 msec 2024-11-21T11:14:36,203 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,203 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,205 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,205 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,206 INFO [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6f9184454915671110d8238a3b71824 columnFamilyName family 2024-11-21T11:14:36,207 DEBUG [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,208 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,208 INFO [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] regionserver.HStore(327): Store=c6f9184454915671110d8238a3b71824/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,208 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,209 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,212 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/fab4371e299cd292e600af4717485beb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,213 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,213 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened fab4371e299cd292e600af4717485beb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61788501, jitterRate=-0.0792795866727829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,213 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:36,213 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for fab4371e299cd292e600af4717485beb: Running coprocessor pre-open hook at 1732187676175Writing region info on filesystem at 1732187676175Initializing all the Stores at 1732187676178 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676178Cleaning up temporary data from old regions at 1732187676193 (+15 ms)Running coprocessor post-open hooks at 1732187676213 (+20 ms)Region opened successfully at 1732187676213 2024-11-21T11:14:36,214 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,214 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/be5404b662521c75e774e2749bf34474/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,214 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,215 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732187674423.fab4371e299cd292e600af4717485beb., pid=51, masterSystemTime=1732187676077 2024-11-21T11:14:36,215 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened be5404b662521c75e774e2749bf34474; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58765064, jitterRate=-0.12433230876922607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,215 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:36,215 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for be5404b662521c75e774e2749bf34474: Running coprocessor pre-open hook at 1732187676181Writing region info on filesystem at 1732187676181Initializing all the Stores at 1732187676182 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676182Cleaning up temporary data from old regions at 1732187676205 (+23 ms)Running coprocessor post-open hooks at 1732187676215 (+10 ms)Region opened successfully at 1732187676215 2024-11-21T11:14:36,217 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732187674423.be5404b662521c75e774e2749bf34474., pid=54, masterSystemTime=1732187676149 2024-11-21T11:14:36,218 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,220 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:36,220 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:36,221 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:36,221 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 46a04157c564adf02211263396700675, NAME => 'test,uuu,1732187674423.46a04157c564adf02211263396700675.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-21T11:14:36,221 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 46a04157c564adf02211263396700675 2024-11-21T11:14:36,221 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,uuu,1732187674423.46a04157c564adf02211263396700675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,221 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,222 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=fab4371e299cd292e600af4717485beb, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,223 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/c6f9184454915671110d8238a3b71824/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,223 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened c6f9184454915671110d8238a3b71824; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58768256, jitterRate=-0.12428474426269531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,224 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:36,224 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for c6f9184454915671110d8238a3b71824: Running coprocessor pre-open hook at 1732187676185Writing region info on filesystem at 1732187676185Initializing all the Stores at 1732187676188 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676188Cleaning up temporary data from old regions at 1732187676214 (+26 ms)Running coprocessor post-open hooks at 1732187676224 (+10 ms)Region opened successfully at 1732187676224 2024-11-21T11:14:36,225 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:36,225 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:36,225 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:36,225 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 075843f8fc377ab19ddd09be30eeaa38, NAME => 'test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-21T11:14:36,226 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,226 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,226 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,226 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,226 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732187674423.c6f9184454915671110d8238a3b71824., pid=35, masterSystemTime=1732187676012 2024-11-21T11:14:36,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure fab4371e299cd292e600af4717485beb, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=be5404b662521c75e774e2749bf34474, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,230 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:36,230 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:36,231 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:36,231 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 05dba68ed06ee6ea199d777ec417cb17, NAME => 'test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-21T11:14:36,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure be5404b662521c75e774e2749bf34474, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,231 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,231 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,231 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,231 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c6f9184454915671110d8238a3b71824, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,235 INFO [StoreOpener-46a04157c564adf02211263396700675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 46a04157c564adf02211263396700675 2024-11-21T11:14:36,237 INFO [StoreOpener-46a04157c564adf02211263396700675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46a04157c564adf02211263396700675 columnFamilyName family 2024-11-21T11:14:36,238 DEBUG [StoreOpener-46a04157c564adf02211263396700675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,237 INFO [StoreOpener-075843f8fc377ab19ddd09be30eeaa38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,239 INFO [StoreOpener-46a04157c564adf02211263396700675-1 {}] regionserver.HStore(327): Store=46a04157c564adf02211263396700675/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,239 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,241 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/46a04157c564adf02211263396700675 2024-11-21T11:14:36,242 INFO [StoreOpener-075843f8fc377ab19ddd09be30eeaa38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075843f8fc377ab19ddd09be30eeaa38 columnFamilyName family 2024-11-21T11:14:36,242 DEBUG [StoreOpener-075843f8fc377ab19ddd09be30eeaa38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,242 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/46a04157c564adf02211263396700675 2024-11-21T11:14:36,243 INFO [StoreOpener-075843f8fc377ab19ddd09be30eeaa38-1 {}] regionserver.HStore(327): Store=075843f8fc377ab19ddd09be30eeaa38/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,243 INFO [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,243 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,243 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,243 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,244 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=8 2024-11-21T11:14:36,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure be5404b662521c75e774e2749bf34474, server=7b462513bfc2,33361,1732187671172 in 186 msec 2024-11-21T11:14:36,245 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,246 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,246 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=be5404b662521c75e774e2749bf34474, ASSIGN in 588 msec 2024-11-21T11:14:36,249 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7b462513bfc2,44785,1732187671410, table=test, region=fab4371e299cd292e600af4717485beb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T11:14:36,250 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7b462513bfc2,45537,1732187671328, table=test, region=c6f9184454915671110d8238a3b71824. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T11:14:36,250 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,252 INFO [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05dba68ed06ee6ea199d777ec417cb17 columnFamilyName family 2024-11-21T11:14:36,253 DEBUG [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,253 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,256 INFO [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] regionserver.HStore(327): Store=05dba68ed06ee6ea199d777ec417cb17/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,259 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,260 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=9 2024-11-21T11:14:36,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,45537,1732187671328 in 381 msec 2024-11-21T11:14:36,261 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,261 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,261 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN in 604 msec 2024-11-21T11:14:36,265 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=6 2024-11-21T11:14:36,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure fab4371e299cd292e600af4717485beb, server=7b462513bfc2,44785,1732187671410 in 243 msec 2024-11-21T11:14:36,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fab4371e299cd292e600af4717485beb, ASSIGN in 612 msec 2024-11-21T11:14:36,277 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/075843f8fc377ab19ddd09be30eeaa38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,278 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/46a04157c564adf02211263396700675/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,278 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 075843f8fc377ab19ddd09be30eeaa38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74977498, jitterRate=0.11725178360939026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,278 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:36,278 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 075843f8fc377ab19ddd09be30eeaa38: Running coprocessor pre-open hook at 1732187676226Writing region info on filesystem at 1732187676226Initializing all the Stores at 1732187676229 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676229Cleaning up temporary data from old regions at 1732187676246 (+17 ms)Running coprocessor post-open hooks at 1732187676278 (+32 ms)Region opened successfully at 1732187676278 2024-11-21T11:14:36,278 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 46a04157c564adf02211263396700675; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61531520, jitterRate=-0.08310890197753906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,278 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46a04157c564adf02211263396700675 2024-11-21T11:14:36,279 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 46a04157c564adf02211263396700675: Running coprocessor pre-open hook at 1732187676222Writing region info on filesystem at 1732187676222Initializing all the Stores at 1732187676223 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676223Cleaning up temporary data from old regions at 1732187676243 (+20 ms)Running coprocessor post-open hooks at 1732187676279 (+36 ms)Region opened successfully at 1732187676279 2024-11-21T11:14:36,281 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732187674423.46a04157c564adf02211263396700675., pid=44, masterSystemTime=1732187676077 2024-11-21T11:14:36,281 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38., pid=55, masterSystemTime=1732187676149 2024-11-21T11:14:36,281 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/05dba68ed06ee6ea199d777ec417cb17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,282 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 05dba68ed06ee6ea199d777ec417cb17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68190140, jitterRate=0.01611226797103882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,282 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:36,282 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 05dba68ed06ee6ea199d777ec417cb17: Running coprocessor pre-open hook at 1732187676232Writing region info on filesystem at 1732187676232Initializing all the Stores at 1732187676237 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676237Cleaning up temporary data from old regions at 1732187676262 (+25 ms)Running coprocessor post-open hooks at 1732187676282 (+20 ms)Region opened successfully at 1732187676282 2024-11-21T11:14:36,283 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17., pid=31, masterSystemTime=1732187676012 2024-11-21T11:14:36,285 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:36,285 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:36,285 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => feb12ac653343cadcf32bc8dff1d6846, NAME => 'test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:36,286 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,286 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f738404802b7de06817afff11fed6aa, NAME => 'test,sss,1732187674423.8f738404802b7de06817afff11fed6aa.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=46a04157c564adf02211263396700675, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,286 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,287 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,287 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,sss,1732187674423.8f738404802b7de06817afff11fed6aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,287 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,287 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,289 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:36,289 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:36,289 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:36,289 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 78487f45e051e2d9f1275ac2ef77493d, NAME => 'test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-21T11:14:36,289 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,289 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,289 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,290 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46a04157c564adf02211263396700675, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,292 INFO [StoreOpener-8f738404802b7de06817afff11fed6aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,295 INFO [StoreOpener-8f738404802b7de06817afff11fed6aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f738404802b7de06817afff11fed6aa columnFamilyName family 2024-11-21T11:14:36,295 DEBUG [StoreOpener-8f738404802b7de06817afff11fed6aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=25 2024-11-21T11:14:36,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 46a04157c564adf02211263396700675, server=7b462513bfc2,44785,1732187671410 in 331 msec 2024-11-21T11:14:36,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=46a04157c564adf02211263396700675, ASSIGN in 640 msec 2024-11-21T11:14:36,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=05dba68ed06ee6ea199d777ec417cb17, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,301 INFO [StoreOpener-8f738404802b7de06817afff11fed6aa-1 {}] regionserver.HStore(327): Store=8f738404802b7de06817afff11fed6aa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,302 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,303 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,303 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,304 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,304 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,307 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,308 INFO [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,310 INFO [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78487f45e051e2d9f1275ac2ef77493d columnFamilyName family 2024-11-21T11:14:36,311 DEBUG [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,311 INFO [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] regionserver.HStore(327): Store=78487f45e051e2d9f1275ac2ef77493d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,311 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,313 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,314 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,315 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,315 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,317 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,320 INFO [StoreOpener-feb12ac653343cadcf32bc8dff1d6846-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 075843f8fc377ab19ddd09be30eeaa38, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,323 INFO [StoreOpener-feb12ac653343cadcf32bc8dff1d6846-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region feb12ac653343cadcf32bc8dff1d6846 columnFamilyName family 2024-11-21T11:14:36,323 DEBUG [StoreOpener-feb12ac653343cadcf32bc8dff1d6846-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,324 INFO [StoreOpener-feb12ac653343cadcf32bc8dff1d6846-1 {}] regionserver.HStore(327): Store=feb12ac653343cadcf32bc8dff1d6846/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,324 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,325 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=27 2024-11-21T11:14:36,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,45537,1732187671328 in 486 msec 2024-11-21T11:14:36,325 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,327 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,327 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN in 668 msec 2024-11-21T11:14:36,330 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,334 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/78487f45e051e2d9f1275ac2ef77493d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,334 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 78487f45e051e2d9f1275ac2ef77493d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73506814, jitterRate=0.09533688426017761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,335 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:36,335 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 78487f45e051e2d9f1275ac2ef77493d: Running coprocessor pre-open hook at 1732187676290Writing region info on filesystem at 1732187676290Initializing all the Stores at 1732187676291 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676291Cleaning up temporary data from old regions at 1732187676315 (+24 ms)Running coprocessor post-open hooks at 1732187676335 (+20 ms)Region opened successfully at 1732187676335 2024-11-21T11:14:36,336 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d., pid=34, masterSystemTime=1732187676012 2024-11-21T11:14:36,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=11 2024-11-21T11:14:36,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 075843f8fc377ab19ddd09be30eeaa38, server=7b462513bfc2,33361,1732187671172 in 273 msec 2024-11-21T11:14:36,339 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:36,339 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:36,339 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:36,339 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => aa29bf2ce1b3154b66b07cc896f4188a, NAME => 'test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T11:14:36,339 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,339 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,340 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,340 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=075843f8fc377ab19ddd09be30eeaa38, ASSIGN in 680 msec 2024-11-21T11:14:36,343 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=78487f45e051e2d9f1275ac2ef77493d, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,354 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/8f738404802b7de06817afff11fed6aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,355 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 8f738404802b7de06817afff11fed6aa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60753385, jitterRate=-0.09470401704311371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,355 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:36,356 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 8f738404802b7de06817afff11fed6aa: Running coprocessor pre-open hook at 1732187676287Writing region info on filesystem at 1732187676287Initializing all the Stores at 1732187676288 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676289 (+1 ms)Cleaning up temporary data from old regions at 1732187676304 (+15 ms)Running coprocessor post-open hooks at 1732187676356 (+52 ms)Region opened successfully at 1732187676356 2024-11-21T11:14:36,361 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732187674423.8f738404802b7de06817afff11fed6aa., pid=47, masterSystemTime=1732187676149 2024-11-21T11:14:36,365 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:36,365 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:36,365 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:36,366 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 325cd18a1a1b735062fa7f51132edef4, NAME => 'test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-21T11:14:36,366 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,366 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,366 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,366 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=7 2024-11-21T11:14:36,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,45537,1732187671328 in 483 msec 2024-11-21T11:14:36,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN in 713 msec 2024-11-21T11:14:36,373 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/feb12ac653343cadcf32bc8dff1d6846/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,374 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened feb12ac653343cadcf32bc8dff1d6846; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68593880, jitterRate=0.02212846279144287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,375 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:36,375 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for feb12ac653343cadcf32bc8dff1d6846: Running coprocessor pre-open hook at 1732187676287Writing region info on filesystem at 1732187676287Initializing all the Stores at 1732187676293 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676293Cleaning up temporary data from old regions at 1732187676327 (+34 ms)Running coprocessor post-open hooks at 1732187676375 (+48 ms)Region opened successfully at 1732187676375 2024-11-21T11:14:36,378 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846., pid=42, masterSystemTime=1732187676077 2024-11-21T11:14:36,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8f738404802b7de06817afff11fed6aa, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f738404802b7de06817afff11fed6aa, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,382 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:36,382 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:36,382 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:36,383 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 76043d320d93c5a6c1b870931771d01c, NAME => 'test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-21T11:14:36,383 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,383 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,383 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,383 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=23 2024-11-21T11:14:36,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f738404802b7de06817afff11fed6aa, server=7b462513bfc2,33361,1732187671172 in 389 msec 2024-11-21T11:14:36,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8f738404802b7de06817afff11fed6aa, ASSIGN in 732 msec 2024-11-21T11:14:36,392 INFO [StoreOpener-325cd18a1a1b735062fa7f51132edef4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,395 INFO [StoreOpener-325cd18a1a1b735062fa7f51132edef4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 325cd18a1a1b735062fa7f51132edef4 columnFamilyName family 2024-11-21T11:14:36,395 DEBUG [StoreOpener-325cd18a1a1b735062fa7f51132edef4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,396 INFO [StoreOpener-325cd18a1a1b735062fa7f51132edef4-1 {}] regionserver.HStore(327): Store=325cd18a1a1b735062fa7f51132edef4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,396 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,397 INFO [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,397 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure feb12ac653343cadcf32bc8dff1d6846, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,399 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,400 INFO [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa29bf2ce1b3154b66b07cc896f4188a columnFamilyName family 2024-11-21T11:14:36,400 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,400 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,400 DEBUG [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,401 INFO [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] regionserver.HStore(327): Store=aa29bf2ce1b3154b66b07cc896f4188a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,402 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,403 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,403 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,404 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,404 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,404 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,407 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=13 2024-11-21T11:14:36,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure feb12ac653343cadcf32bc8dff1d6846, server=7b462513bfc2,44785,1732187671410 in 461 msec 2024-11-21T11:14:36,420 INFO [StoreOpener-76043d320d93c5a6c1b870931771d01c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,421 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/325cd18a1a1b735062fa7f51132edef4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=feb12ac653343cadcf32bc8dff1d6846, ASSIGN in 761 msec 2024-11-21T11:14:36,422 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 325cd18a1a1b735062fa7f51132edef4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68845655, jitterRate=0.025880202651023865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,422 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:36,422 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 325cd18a1a1b735062fa7f51132edef4: Running coprocessor pre-open hook at 1732187676366Writing region info on filesystem at 1732187676366Initializing all the Stores at 1732187676368 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676368Cleaning up temporary data from old regions at 1732187676400 (+32 ms)Running coprocessor post-open hooks at 1732187676422 (+22 ms)Region opened successfully at 1732187676422 2024-11-21T11:14:36,424 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4., pid=49, masterSystemTime=1732187676149 2024-11-21T11:14:36,425 INFO [StoreOpener-76043d320d93c5a6c1b870931771d01c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76043d320d93c5a6c1b870931771d01c columnFamilyName family 2024-11-21T11:14:36,425 DEBUG [StoreOpener-76043d320d93c5a6c1b870931771d01c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,426 INFO [StoreOpener-76043d320d93c5a6c1b870931771d01c-1 {}] regionserver.HStore(327): Store=76043d320d93c5a6c1b870931771d01c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,427 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,428 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,429 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/aa29bf2ce1b3154b66b07cc896f4188a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,429 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,429 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:36,429 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:36,429 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:36,429 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => d2b570ae93fb4b7f17a9d44dce21589f, NAME => 'test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-21T11:14:36,430 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,430 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,430 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,430 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,431 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened aa29bf2ce1b3154b66b07cc896f4188a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64524394, jitterRate=-0.03851160407066345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for aa29bf2ce1b3154b66b07cc896f4188a: Running coprocessor pre-open hook at 1732187676340Writing region info on filesystem at 1732187676340Initializing all the Stores at 1732187676342 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676342Cleaning up temporary data from old regions at 1732187676404 (+62 ms)Running coprocessor post-open hooks at 1732187676431 (+27 ms)Region opened successfully at 1732187676431 2024-11-21T11:14:36,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,433 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a., pid=38, masterSystemTime=1732187676012 2024-11-21T11:14:36,436 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,440 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:36,440 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:36,440 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:36,440 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 42b6f7e93058e77d80053f0460a70897, NAME => 'test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-21T11:14:36,441 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,441 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,441 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,441 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 325cd18a1a1b735062fa7f51132edef4, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,446 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/76043d320d93c5a6c1b870931771d01c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,446 INFO [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,447 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 76043d320d93c5a6c1b870931771d01c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74149121, jitterRate=0.10490800440311432}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,447 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:36,448 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 76043d320d93c5a6c1b870931771d01c: Running coprocessor pre-open hook at 1732187676383Writing region info on filesystem at 1732187676383Initializing all the Stores at 1732187676385 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676385Cleaning up temporary data from old regions at 1732187676431 (+46 ms)Running coprocessor post-open hooks at 1732187676448 (+17 ms)Region opened successfully at 1732187676448 2024-11-21T11:14:36,448 INFO [StoreOpener-d2b570ae93fb4b7f17a9d44dce21589f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=30 2024-11-21T11:14:36,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 325cd18a1a1b735062fa7f51132edef4, server=7b462513bfc2,33361,1732187671172 in 437 msec 2024-11-21T11:14:36,451 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c., pid=46, masterSystemTime=1732187676077 2024-11-21T11:14:36,451 INFO [StoreOpener-d2b570ae93fb4b7f17a9d44dce21589f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b570ae93fb4b7f17a9d44dce21589f columnFamilyName family 2024-11-21T11:14:36,451 DEBUG [StoreOpener-d2b570ae93fb4b7f17a9d44dce21589f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,452 INFO [StoreOpener-d2b570ae93fb4b7f17a9d44dce21589f-1 {}] regionserver.HStore(327): Store=d2b570ae93fb4b7f17a9d44dce21589f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,452 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,453 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,454 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,455 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,455 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,460 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:36,460 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:36,460 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:36,460 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 993840b9e61b5baf1c5a77b64bde7e9d, NAME => 'test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-21T11:14:36,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=325cd18a1a1b735062fa7f51132edef4, ASSIGN in 793 msec 2024-11-21T11:14:36,461 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,461 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,461 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,461 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,462 INFO [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b6f7e93058e77d80053f0460a70897 columnFamilyName family 2024-11-21T11:14:36,463 DEBUG [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,465 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,466 INFO [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] regionserver.HStore(327): Store=42b6f7e93058e77d80053f0460a70897/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,466 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,467 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,468 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76043d320d93c5a6c1b870931771d01c, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,474 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-11-21T11:14:36,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 76043d320d93c5a6c1b870931771d01c, server=7b462513bfc2,44785,1732187671410 in 493 msec 2024-11-21T11:14:36,479 INFO [StoreOpener-993840b9e61b5baf1c5a77b64bde7e9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=76043d320d93c5a6c1b870931771d01c, ASSIGN in 821 msec 2024-11-21T11:14:36,481 INFO [StoreOpener-993840b9e61b5baf1c5a77b64bde7e9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 993840b9e61b5baf1c5a77b64bde7e9d columnFamilyName family 2024-11-21T11:14:36,481 DEBUG [StoreOpener-993840b9e61b5baf1c5a77b64bde7e9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,482 INFO [StoreOpener-993840b9e61b5baf1c5a77b64bde7e9d-1 {}] regionserver.HStore(327): Store=993840b9e61b5baf1c5a77b64bde7e9d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,482 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,483 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,484 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,484 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,484 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=22 2024-11-21T11:14:36,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,45537,1732187671328 in 586 msec 2024-11-21T11:14:36,487 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN in 829 msec 2024-11-21T11:14:36,497 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/42b6f7e93058e77d80053f0460a70897/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,498 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/d2b570ae93fb4b7f17a9d44dce21589f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,498 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 42b6f7e93058e77d80053f0460a70897; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61020917, jitterRate=-0.09071747958660126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,499 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:36,499 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 42b6f7e93058e77d80053f0460a70897: Running coprocessor pre-open hook at 1732187676441Writing region info on filesystem at 1732187676441Initializing all the Stores at 1732187676443 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676443Cleaning up temporary data from old regions at 1732187676469 (+26 ms)Running coprocessor post-open hooks at 1732187676499 (+30 ms)Region opened successfully at 1732187676499 2024-11-21T11:14:36,499 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened d2b570ae93fb4b7f17a9d44dce21589f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75387621, jitterRate=0.12336309254169464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,499 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for d2b570ae93fb4b7f17a9d44dce21589f: Running coprocessor pre-open hook at 1732187676430Writing region info on filesystem at 1732187676430Initializing all the Stores at 1732187676435 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676436 (+1 ms)Cleaning up temporary data from old regions at 1732187676455 (+19 ms)Running coprocessor post-open hooks at 1732187676499 (+44 ms)Region opened successfully at 1732187676499 2024-11-21T11:14:36,500 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897., pid=33, masterSystemTime=1732187676012 2024-11-21T11:14:36,502 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f., pid=52, masterSystemTime=1732187676149 2024-11-21T11:14:36,505 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:36,505 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:36,505 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:36,505 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f3c54f1f14d06f35d9d402ce20065f9, NAME => 'test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-21T11:14:36,506 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/993840b9e61b5baf1c5a77b64bde7e9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,506 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,506 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,506 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,506 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,506 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 993840b9e61b5baf1c5a77b64bde7e9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60369905, jitterRate=-0.10041831433773041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,506 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:36,507 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 993840b9e61b5baf1c5a77b64bde7e9d: Running coprocessor pre-open hook at 1732187676461Writing region info on filesystem at 1732187676461Initializing all the Stores at 1732187676464 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676464Cleaning up temporary data from old regions at 1732187676484 (+20 ms)Running coprocessor post-open hooks at 1732187676506 (+22 ms)Region opened successfully at 1732187676507 (+1 ms) 2024-11-21T11:14:36,507 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=42b6f7e93058e77d80053f0460a70897, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:36,509 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:36,509 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 9501094853170e6297b53847d78f71c8, NAME => 'test,nnn,1732187674423.9501094853170e6297b53847d78f71c8.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-21T11:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,nnn,1732187674423.9501094853170e6297b53847d78f71c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,510 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,510 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,510 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d., pid=45, masterSystemTime=1732187676077 2024-11-21T11:14:36,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,514 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:36,514 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:36,515 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:36,515 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f15a03020943182be6b4e66ea5cf19a, NAME => 'test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-21T11:14:36,515 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,515 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,516 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,516 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b570ae93fb4b7f17a9d44dce21589f, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,522 INFO [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 993840b9e61b5baf1c5a77b64bde7e9d, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,525 INFO [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3c54f1f14d06f35d9d402ce20065f9 columnFamilyName family 2024-11-21T11:14:36,525 DEBUG [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,526 INFO [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] regionserver.HStore(327): Store=6f3c54f1f14d06f35d9d402ce20065f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,526 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,527 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,527 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,528 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,528 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=21 2024-11-21T11:14:36,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure d2b570ae93fb4b7f17a9d44dce21589f, server=7b462513bfc2,33361,1732187671172 in 485 msec 2024-11-21T11:14:36,531 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d2b570ae93fb4b7f17a9d44dce21589f, ASSIGN in 873 msec 2024-11-21T11:14:36,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=28 2024-11-21T11:14:36,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 993840b9e61b5baf1c5a77b64bde7e9d, server=7b462513bfc2,44785,1732187671410 in 564 msec 2024-11-21T11:14:36,533 INFO [StoreOpener-9501094853170e6297b53847d78f71c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,541 INFO [StoreOpener-3f15a03020943182be6b4e66ea5cf19a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,542 INFO [StoreOpener-9501094853170e6297b53847d78f71c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9501094853170e6297b53847d78f71c8 columnFamilyName family 2024-11-21T11:14:36,543 DEBUG [StoreOpener-9501094853170e6297b53847d78f71c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=993840b9e61b5baf1c5a77b64bde7e9d, ASSIGN in 881 msec 2024-11-21T11:14:36,545 INFO [StoreOpener-3f15a03020943182be6b4e66ea5cf19a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f15a03020943182be6b4e66ea5cf19a columnFamilyName family 2024-11-21T11:14:36,545 DEBUG [StoreOpener-3f15a03020943182be6b4e66ea5cf19a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,547 INFO [StoreOpener-9501094853170e6297b53847d78f71c8-1 {}] regionserver.HStore(327): Store=9501094853170e6297b53847d78f71c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,549 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/6f3c54f1f14d06f35d9d402ce20065f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,549 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,549 INFO [StoreOpener-3f15a03020943182be6b4e66ea5cf19a-1 {}] regionserver.HStore(327): Store=3f15a03020943182be6b4e66ea5cf19a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,549 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,550 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 6f3c54f1f14d06f35d9d402ce20065f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69897029, jitterRate=0.04154689610004425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,550 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:36,550 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 6f3c54f1f14d06f35d9d402ce20065f9: Running coprocessor pre-open hook at 1732187676506Writing region info on filesystem at 1732187676506Initializing all the Stores at 1732187676511 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676511Cleaning up temporary data from old regions at 1732187676528 (+17 ms)Running coprocessor post-open hooks at 1732187676550 (+22 ms)Region opened successfully at 1732187676550 2024-11-21T11:14:36,550 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,550 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,551 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,551 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,552 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,552 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,552 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,552 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,552 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9., pid=36, masterSystemTime=1732187676012 2024-11-21T11:14:36,555 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,555 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=12 2024-11-21T11:14:36,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,45537,1732187671328 in 689 msec 2024-11-21T11:14:36,555 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:36,555 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:36,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN in 898 msec 2024-11-21T11:14:36,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:36,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:36,571 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9501094853170e6297b53847d78f71c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,572 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 9501094853170e6297b53847d78f71c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73250752, jitterRate=0.0915212631225586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,573 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:36,573 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 9501094853170e6297b53847d78f71c8: Running coprocessor pre-open hook at 1732187676510Writing region info on filesystem at 1732187676510Initializing all the Stores at 1732187676512 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676512Cleaning up temporary data from old regions at 1732187676552 (+40 ms)Running coprocessor post-open hooks at 1732187676573 (+21 ms)Region opened successfully at 1732187676573 2024-11-21T11:14:36,574 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732187674423.9501094853170e6297b53847d78f71c8., pid=56, masterSystemTime=1732187676149 2024-11-21T11:14:36,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=15 2024-11-21T11:14:36,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,45537,1732187671328 in 688 msec 2024-11-21T11:14:36,578 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:36,578 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:36,579 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/3f15a03020943182be6b4e66ea5cf19a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ad652628fe53a6af579af31b1d64bdf, NAME => 'test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-21T11:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,580 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,579 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 3f15a03020943182be6b4e66ea5cf19a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67551352, jitterRate=0.006593585014343262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,580 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:36,580 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 3f15a03020943182be6b4e66ea5cf19a: Running coprocessor pre-open hook at 1732187676516Writing region info on filesystem at 1732187676516Initializing all the Stores at 1732187676517 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676517Cleaning up temporary data from old regions at 1732187676552 (+35 ms)Running coprocessor post-open hooks at 1732187676580 (+28 ms)Region opened successfully at 1732187676580 2024-11-21T11:14:36,581 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a., pid=50, masterSystemTime=1732187676077 2024-11-21T11:14:36,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN in 920 msec 2024-11-21T11:14:36,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=9501094853170e6297b53847d78f71c8, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,585 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:36,585 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:36,585 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:36,585 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => ad417eadf023765c2d8ea0936fcb018c, NAME => 'test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-21T11:14:36,586 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,586 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,586 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,586 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9501094853170e6297b53847d78f71c8, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f15a03020943182be6b4e66ea5cf19a, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=18 2024-11-21T11:14:36,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 9501094853170e6297b53847d78f71c8, server=7b462513bfc2,33361,1732187671172 in 531 msec 2024-11-21T11:14:36,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9501094853170e6297b53847d78f71c8, ASSIGN in 940 msec 2024-11-21T11:14:36,604 INFO [StoreOpener-5ad652628fe53a6af579af31b1d64bdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=5 2024-11-21T11:14:36,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f15a03020943182be6b4e66ea5cf19a, server=7b462513bfc2,44785,1732187671410 in 586 msec 2024-11-21T11:14:36,605 INFO [StoreOpener-ad417eadf023765c2d8ea0936fcb018c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,607 INFO [StoreOpener-5ad652628fe53a6af579af31b1d64bdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ad652628fe53a6af579af31b1d64bdf columnFamilyName family 2024-11-21T11:14:36,607 DEBUG [StoreOpener-5ad652628fe53a6af579af31b1d64bdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3f15a03020943182be6b4e66ea5cf19a, ASSIGN in 948 msec 2024-11-21T11:14:36,609 INFO [StoreOpener-5ad652628fe53a6af579af31b1d64bdf-1 {}] regionserver.HStore(327): Store=5ad652628fe53a6af579af31b1d64bdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,609 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,610 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,612 INFO [StoreOpener-ad417eadf023765c2d8ea0936fcb018c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad417eadf023765c2d8ea0936fcb018c columnFamilyName family 2024-11-21T11:14:36,612 DEBUG [StoreOpener-ad417eadf023765c2d8ea0936fcb018c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,614 INFO [StoreOpener-ad417eadf023765c2d8ea0936fcb018c-1 {}] regionserver.HStore(327): Store=ad417eadf023765c2d8ea0936fcb018c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,614 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,615 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,615 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,616 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,617 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,617 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,620 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/5ad652628fe53a6af579af31b1d64bdf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,622 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 5ad652628fe53a6af579af31b1d64bdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60341035, jitterRate=-0.10084851086139679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,622 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:36,622 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 5ad652628fe53a6af579af31b1d64bdf: Running coprocessor pre-open hook at 1732187676580Writing region info on filesystem at 1732187676580Initializing all the Stores at 1732187676582 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676582Cleaning up temporary data from old regions at 1732187676611 (+29 ms)Running coprocessor post-open hooks at 1732187676622 (+11 ms)Region opened successfully at 1732187676622 2024-11-21T11:14:36,622 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,623 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf., pid=48, masterSystemTime=1732187676149 2024-11-21T11:14:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:36,635 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:36,635 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:36,635 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:36,636 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 06bb2d53bc41f5595f048fbf33d2e7f6, NAME => 'test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-21T11:14:36,636 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,636 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,636 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,636 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,636 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ad652628fe53a6af579af31b1d64bdf, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=26 2024-11-21T11:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ad417eadf023765c2d8ea0936fcb018c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ad652628fe53a6af579af31b1d64bdf, server=7b462513bfc2,33361,1732187671172 in 643 msec 2024-11-21T11:14:36,650 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened ad417eadf023765c2d8ea0936fcb018c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71145235, jitterRate=0.06014661490917206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,650 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:36,651 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for ad417eadf023765c2d8ea0936fcb018c: Running coprocessor pre-open hook at 1732187676586Writing region info on filesystem at 1732187676586Initializing all the Stores at 1732187676588 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676588Cleaning up temporary data from old regions at 1732187676617 (+29 ms)Running coprocessor post-open hooks at 1732187676651 (+34 ms)Region opened successfully at 1732187676651 2024-11-21T11:14:36,653 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c., pid=41, masterSystemTime=1732187676077 2024-11-21T11:14:36,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5ad652628fe53a6af579af31b1d64bdf, ASSIGN in 993 msec 2024-11-21T11:14:36,655 INFO [StoreOpener-06bb2d53bc41f5595f048fbf33d2e7f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,659 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:36,659 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:36,659 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:36,659 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => ea0e500737edd661e0f9e38b0cf0f99a, NAME => 'test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-21T11:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,665 INFO [StoreOpener-06bb2d53bc41f5595f048fbf33d2e7f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06bb2d53bc41f5595f048fbf33d2e7f6 columnFamilyName family 2024-11-21T11:14:36,665 DEBUG [StoreOpener-06bb2d53bc41f5595f048fbf33d2e7f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,665 INFO [StoreOpener-ea0e500737edd661e0f9e38b0cf0f99a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad417eadf023765c2d8ea0936fcb018c, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,667 INFO [StoreOpener-06bb2d53bc41f5595f048fbf33d2e7f6-1 {}] regionserver.HStore(327): Store=06bb2d53bc41f5595f048fbf33d2e7f6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,668 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,669 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,672 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,673 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,673 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=17 2024-11-21T11:14:36,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ad417eadf023765c2d8ea0936fcb018c, server=7b462513bfc2,44785,1732187671410 in 733 msec 2024-11-21T11:14:36,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ad417eadf023765c2d8ea0936fcb018c, ASSIGN in 1.0170 sec 2024-11-21T11:14:36,678 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,680 INFO [StoreOpener-ea0e500737edd661e0f9e38b0cf0f99a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea0e500737edd661e0f9e38b0cf0f99a columnFamilyName family 2024-11-21T11:14:36,680 DEBUG [StoreOpener-ea0e500737edd661e0f9e38b0cf0f99a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:36,683 INFO [StoreOpener-ea0e500737edd661e0f9e38b0cf0f99a-1 {}] regionserver.HStore(327): Store=ea0e500737edd661e0f9e38b0cf0f99a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:36,684 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,684 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,685 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,686 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,686 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,688 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,694 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/06bb2d53bc41f5595f048fbf33d2e7f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,695 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 06bb2d53bc41f5595f048fbf33d2e7f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65416717, jitterRate=-0.02521495521068573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,695 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:36,695 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 06bb2d53bc41f5595f048fbf33d2e7f6: Running coprocessor pre-open hook at 1732187676637Writing region info on filesystem at 1732187676637Initializing all the Stores at 1732187676640 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676640Cleaning up temporary data from old regions at 1732187676673 (+33 ms)Running coprocessor post-open hooks at 1732187676695 (+22 ms)Region opened successfully at 1732187676695 2024-11-21T11:14:36,698 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6., pid=53, masterSystemTime=1732187676149 2024-11-21T11:14:36,702 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:36,702 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:36,703 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ea0e500737edd661e0f9e38b0cf0f99a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T11:14:36,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:36,704 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened ea0e500737edd661e0f9e38b0cf0f99a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70495193, jitterRate=0.05046023428440094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:36,705 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:36,705 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for ea0e500737edd661e0f9e38b0cf0f99a: Running coprocessor pre-open hook at 1732187676660Writing region info on filesystem at 1732187676660Initializing all the Stores at 1732187676662 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187676662Cleaning up temporary data from old regions at 1732187676686 (+24 ms)Running coprocessor post-open hooks at 1732187676705 (+19 ms)Region opened successfully at 1732187676705 2024-11-21T11:14:36,707 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a., pid=43, masterSystemTime=1732187676077 2024-11-21T11:14:36,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06bb2d53bc41f5595f048fbf33d2e7f6, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:36,710 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:36,710 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:36,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=14 2024-11-21T11:14:36,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 06bb2d53bc41f5595f048fbf33d2e7f6, server=7b462513bfc2,33361,1732187671172 in 663 msec 2024-11-21T11:14:36,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=06bb2d53bc41f5595f048fbf33d2e7f6, ASSIGN in 1.0600 sec 2024-11-21T11:14:36,722 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, openSeqNum=2, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:36,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea0e500737edd661e0f9e38b0cf0f99a, server=7b462513bfc2,44785,1732187671410 because future has completed 2024-11-21T11:14:36,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=20 2024-11-21T11:14:36,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ea0e500737edd661e0f9e38b0cf0f99a, server=7b462513bfc2,44785,1732187671410 in 772 msec 2024-11-21T11:14:36,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=4 2024-11-21T11:14:36,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ea0e500737edd661e0f9e38b0cf0f99a, ASSIGN in 1.0780 sec 2024-11-21T11:14:36,744 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T11:14:36,745 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732187676745"}]},"ts":"1732187676745"} 2024-11-21T11:14:36,751 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-21T11:14:36,753 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T11:14:36,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 2.3190 sec 2024-11-21T11:14:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T11:14:38,645 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-21T11:14:38,645 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-21T11:14:38,647 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T11:14:38,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-21T11:14:38,666 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T11:14:38,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-21T11:14:38,668 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T11:14:38,673 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T11:14:38,696 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T11:14:38,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:38,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:38,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:38,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region be5404b662521c75e774e2749bf34474 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 075843f8fc377ab19ddd09be30eeaa38 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 06bb2d53bc41f5595f048fbf33d2e7f6 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9501094853170e6297b53847d78f71c8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d2b570ae93fb4b7f17a9d44dce21589f changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8f738404802b7de06817afff11fed6aa changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5ad652628fe53a6af579af31b1d64bdf changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 325cd18a1a1b735062fa7f51132edef4 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3f15a03020943182be6b4e66ea5cf19a changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fab4371e299cd292e600af4717485beb changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 76043d320d93c5a6c1b870931771d01c changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region feb12ac653343cadcf32bc8dff1d6846 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ad417eadf023765c2d8ea0936fcb018c changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ea0e500737edd661e0f9e38b0cf0f99a changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 46a04157c564adf02211263396700675 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 993840b9e61b5baf1c5a77b64bde7e9d changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f52493bc04c75885babe0eeaf6faf9b0 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 78487f45e051e2d9f1275ac2ef77493d changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c6f9184454915671110d8238a3b71824 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 42b6f7e93058e77d80053f0460a70897 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6f3c54f1f14d06f35d9d402ce20065f9 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f1d6b26cdbeafd1feed231ef1797ec16 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb15d5d5ec6aa108fb3f69b646d5757d changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aa29bf2ce1b3154b66b07cc896f4188a changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9c3ba6d97819a96bf36b5d4ae2cc3573 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 05dba68ed06ee6ea199d777ec417cb17 changed from -1.0 to 0.0, refreshing cache 2024-11-21T11:14:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-21T11:14:38,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-21T11:14:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-21T11:14:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(204): Hosts are {7b462513bfc2=0} racks are {/default-rack=0} 2024-11-21T11:14:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-21T11:14:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-21T11:14:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(303): server 2 has 10 regions 2024-11-21T11:14:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T11:14:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T11:14:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T11:14:38,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T11:14:38,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T11:14:38,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T11:14:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-21T11:14:38,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-21T11:14:38,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-21T11:14:39,325 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T11:14:39,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 532 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-21T11:14:39,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-21T11:14:39,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.HMaster(2172): balance hri=eb15d5d5ec6aa108fb3f69b646d5757d, source=7b462513bfc2,45537,1732187671328, destination=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:39,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:39,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=CLOSING, regionLocation=7b462513bfc2,45537,1732187671328 2024-11-21T11:14:39,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE because future has completed 2024-11-21T11:14:39,353 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T11:14:39,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,45537,1732187671328}] 2024-11-21T11:14:39,389 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-21T11:14:39,392 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T11:14:39,513 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,513 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T11:14:39,514 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing eb15d5d5ec6aa108fb3f69b646d5757d, disabling compactions & flushes 2024-11-21T11:14:39,514 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,514 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,514 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. after waiting 0 ms 2024-11-21T11:14:39,514 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,519 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:39,522 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,523 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for eb15d5d5ec6aa108fb3f69b646d5757d: Waiting for close lock at 1732187679514Running coprocessor pre-close hooks at 1732187679514Disabling compacts and flushes for region at 1732187679514Disabling writes for close at 1732187679514Writing region close event to WAL at 1732187679514Running coprocessor post-close hooks at 1732187679520 (+6 ms)Closed at 1732187679522 (+2 ms) 2024-11-21T11:14:39,523 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding eb15d5d5ec6aa108fb3f69b646d5757d move to 7b462513bfc2,33361,1732187671172 record at close sequenceid=2 2024-11-21T11:14:39,526 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=CLOSED 2024-11-21T11:14:39,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,45537,1732187671328 because future has completed 2024-11-21T11:14:39,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T11:14:39,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,45537,1732187671328 in 177 msec 2024-11-21T11:14:39,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE; state=CLOSED, location=7b462513bfc2,33361,1732187671172; forceNewPlan=false, retain=false 2024-11-21T11:14:39,687 INFO [7b462513bfc2:38843 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T11:14:39,688 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:39,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE because future has completed 2024-11-21T11:14:39,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:39,850 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,850 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => eb15d5d5ec6aa108fb3f69b646d5757d, NAME => 'test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-21T11:14:39,850 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,850 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:39,851 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,851 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,853 INFO [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,854 INFO [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb15d5d5ec6aa108fb3f69b646d5757d columnFamilyName family 2024-11-21T11:14:39,854 DEBUG [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:39,855 INFO [StoreOpener-eb15d5d5ec6aa108fb3f69b646d5757d-1 {}] regionserver.HStore(327): Store=eb15d5d5ec6aa108fb3f69b646d5757d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:39,855 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,856 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,857 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,858 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,858 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,860 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,861 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened eb15d5d5ec6aa108fb3f69b646d5757d; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59514533, jitterRate=-0.11316435039043427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:39,861 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:39,861 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for eb15d5d5ec6aa108fb3f69b646d5757d: Running coprocessor pre-open hook at 1732187679851Writing region info on filesystem at 1732187679851Initializing all the Stores at 1732187679852 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187679852Cleaning up temporary data from old regions at 1732187679858 (+6 ms)Running coprocessor post-open hooks at 1732187679861 (+3 ms)Region opened successfully at 1732187679861 2024-11-21T11:14:39,862 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d., pid=59, masterSystemTime=1732187679844 2024-11-21T11:14:39,864 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,864 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:39,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, openSeqNum=5, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:39,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,33361,1732187671172 because future has completed 2024-11-21T11:14:39,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-21T11:14:39,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure eb15d5d5ec6aa108fb3f69b646d5757d, server=7b462513bfc2,33361,1732187671172 in 178 msec 2024-11-21T11:14:39,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE in 531 msec 2024-11-21T11:14:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-21T11:14:39,947 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T11:14:39,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-21T11:14:39,972 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=379, OpenFileDescriptor=637, MaxFileDescriptor=1048576, SystemLoadAverage=573, ProcessCount=11, AvailableMemoryMB=5172 2024-11-21T11:14:39,975 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-21T11:14:39,975 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-21T11:14:39,976 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 7b462513bfc2,45537,1732187671328 2024-11-21T11:14:39,976 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 7b462513bfc2,45537,1732187671328: Simulated kill ***** 2024-11-21T11:14:39,976 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T11:14:39,979 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T11:14:39,981 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T11:14:39,985 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T11:14:39,986 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T11:14:39,995 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 373893728 }, "NonHeapMemoryUsage": { "committed": 155910144, "init": 7667712, "max": -1, "used": 150736416 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "7b462513bfc2", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34327, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 97, "ProcessCallTime_min": 0, "ProcessCallTime_max": 43, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 10, "ProcessCallTime_median": 21, "ProcessCallTime_75th_percentile": 32, "ProcessCallTime_90th_percentile": 38, "ProcessCallTime_95th_percentile": 40, "ProcessCallTime_98th_percentile": 42, "ProcessCallTime_99th_percentile": 42, "ProcessCallTime_99.9th_percentile": 42, "ProcessCallTime_TimeRangeCount_0-1": 97, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 97, "QueueCallTime_min": 0, "QueueCallTime_max": 22, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 5, "QueueCallTime_median": 11, "QueueCallTime_75th_percentile": 16, "QueueCallTime_90th_percentile": 19, "QueueCallTime_95th_percentile": 20, "QueueCallTime_98th_percentile": 21, "QueueCallTime_99th_percentile": 21, "QueueCallTime_99.9th_percentile": 21, "QueueCallTime_TimeRangeCount_0-1": 97, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 97, "TotalCallTime_min": 0, "TotalCallTime_max": 44, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 11, "TotalCallTime_median": 22, "TotalCallTime_75th_percentile": 33, "TotalCallTime_90th_percentile": 39, "TotalCallTime_95th_percentile": 41, "TotalCallTime_98th_percentile": 43, "TotalCallTime_99th_percentile": 43, "TotalCallTime_99.9th_percentile": 43, "TotalCallTime_TimeRangeCount_0-1": 97, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 97, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 704, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 97, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 97, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 320, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 97, "sentBytes": 54703 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "7b462513bfc2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58590", "tag.serverName": "7b462513bfc2,44785,1732187671410", "tag.clusterId": "138f8f64-bbb0-476e-9377-f6147a31aaee", "tag.Context": "regionserver", "tag.Hostname": "7b462513bfc2", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732187671410, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 1, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T11:14:40,000 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38843 {}] master.MasterRpcServices(700): 7b462513bfc2,45537,1732187671328 reported a fatal error: ***** ABORTING region server 7b462513bfc2,45537,1732187671328: Simulated kill ***** 2024-11-21T11:14:40,002 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b462513bfc2,45537,1732187671328' ***** 2024-11-21T11:14:40,002 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-21T11:14:40,002 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T11:14:40,003 INFO [RS:1;7b462513bfc2:45537 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-21T11:14:40,003 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T11:14:40,003 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T11:14:40,003 INFO [RS:1;7b462513bfc2:45537 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-21T11:14:40,003 INFO [RS:1;7b462513bfc2:45537 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T11:14:40,003 INFO [RS:1;7b462513bfc2:45537 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7b462513bfc2:45537. 2024-11-21T11:14:40,003 DEBUG [RS:1;7b462513bfc2:45537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:40,008 DEBUG [RS:1;7b462513bfc2:45537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:40,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741836_1012 (size=33749) 2024-11-21T11:14:40,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741836_1012 (size=33749) 2024-11-21T11:14:40,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741836_1012 (size=33749) 2024-11-21T11:14:40,018 INFO [regionserver/7b462513bfc2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T11:14:40,019 INFO [regionserver/7b462513bfc2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T11:14:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741834_1010 (size=2774) 2024-11-21T11:14:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741834_1010 (size=2774) 2024-11-21T11:14:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741834_1010 (size=2774) 2024-11-21T11:14:40,021 DEBUG [RS:1;7b462513bfc2:45537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:40,021 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:14:40,021 INFO [RS:1;7b462513bfc2:45537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45537 2024-11-21T11:14:40,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b462513bfc2,45537,1732187671328 2024-11-21T11:14:40,029 INFO [RS:1;7b462513bfc2:45537 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T11:14:40,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T11:14:40,031 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b462513bfc2,45537,1732187671328] 2024-11-21T11:14:40,033 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b462513bfc2,45537,1732187671328 already deleted, retry=false 2024-11-21T11:14:40,033 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7b462513bfc2,45537,1732187671328 on 7b462513bfc2,38843,1732187670157 2024-11-21T11:14:40,038 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:40,042 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 7b462513bfc2,45537,1732187671328 (carryingMeta=true) 7b462513bfc2,45537,1732187671328/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@28908101[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T11:14:40,042 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:40,043 INFO [regionserver/7b462513bfc2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:14:40,043 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true, isMeta: true 2024-11-21T11:14:40,046 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting 2024-11-21T11:14:40,047 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7b462513bfc2,45537,1732187671328 WAL count=1, meta=true 2024-11-21T11:14:40,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta}] 2024-11-21T11:14:40,055 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:40,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:40,104 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T11:14:40,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T11:14:40,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45537-0x1013a3c2d4c0002, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T11:14:40,132 INFO [RS:1;7b462513bfc2:45537 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T11:14:40,132 INFO [RS:1;7b462513bfc2:45537 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b462513bfc2,45537,1732187671328; zookeeper connection closed. 2024-11-21T11:14:40,133 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53b2c9ed {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53b2c9ed 2024-11-21T11:14:40,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33361 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-21T11:14:40,254 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, size=33.0 K (33749bytes) 2024-11-21T11:14:40,255 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-21T11:14:40,255 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-21T11:14:40,255 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta 2024-11-21T11:14:40,261 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta after 4ms 2024-11-21T11:14:40,268 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta took 16ms 2024-11-21T11:14:40,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-21T11:14:40,282 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp 2024-11-21T11:14:40,286 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp 2024-11-21T11:14:40,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta so closing down 2024-11-21T11:14:40,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T11:14:40,290 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T11:14:40,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741864_1040 (size=33508) 2024-11-21T11:14:40,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741864_1040 (size=33508) 2024-11-21T11:14:40,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741864_1040 (size=33508) 2024-11-21T11:14:40,304 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-21T11:14:40,306 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp to hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T11:14:40,310 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-21T11:14:40,310 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, journal: Splitting hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, size=33.0 K (33749bytes) at 1732187680254Creating recovered edits writer path=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp at 1732187680286 (+32 ms)Finishing writing output for hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta so closing down at 1732187680287 (+1 ms)3 split writer threads finished at 1732187680290 (+3 ms)Closed recovered edits writer path=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1732187680304 (+14 ms)Rename recovered edits hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta.temp to hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732187680306 (+2 ms)Processed 62 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732187680310 (+4 ms) 2024-11-21T11:14:40,310 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta 2024-11-21T11:14:40,312 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-21T11:14:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-21T11:14:40,322 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta to hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs 2024-11-21T11:14:40,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T11:14:40,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 in 265 msec 2024-11-21T11:14:40,326 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:40,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T11:14:40,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 in 277 msec 2024-11-21T11:14:40,331 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting dir is empty, no logs to split. 2024-11-21T11:14:40,331 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7b462513bfc2,45537,1732187671328 WAL count=0, meta=true 2024-11-21T11:14:40,331 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 7b462513bfc2,45537,1732187671328 WAL splitting is done? wals=0, meta=true 2024-11-21T11:14:40,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T11:14:40,334 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:40,335 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:40,485 DEBUG [7b462513bfc2:38843 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T11:14:40,485 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(204): Hosts are {7b462513bfc2=0} racks are {/default-rack=0} 2024-11-21T11:14:40,486 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T11:14:40,486 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T11:14:40,486 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T11:14:40,486 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T11:14:40,486 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T11:14:40,486 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T11:14:40,486 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T11:14:40,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:40,488 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPENING 2024-11-21T11:14:40,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:40,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:40,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:40,492 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:40,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:40,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:40,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:40,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:40,650 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T11:14:40,650 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T11:14:40,651 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T11:14:40,653 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b462513bfc2%2C44785%2C1732187671410.meta, suffix=.meta, logDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410, archiveDir=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs, maxLogs=32 2024-11-21T11:14:40,667 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410/7b462513bfc2%2C44785%2C1732187671410.meta.1732187680654.meta, exclude list is [], retry=0 2024-11-21T11:14:40,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39687,DS-e82e6422-f18d-4c88-b228-2a7a500b5ad2,DISK] 2024-11-21T11:14:40,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33703,DS-c5c87aee-8613-492a-9420-18b36a7ae926,DISK] 2024-11-21T11:14:40,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33187,DS-9eb6b1ed-0568-4798-bd10-36be5c339628,DISK] 2024-11-21T11:14:40,674 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,44785,1732187671410/7b462513bfc2%2C44785%2C1732187671410.meta.1732187680654.meta 2024-11-21T11:14:40,674 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37171:37171),(127.0.0.1/127.0.0.1:32963:32963),(127.0.0.1/127.0.0.1:39101:39101)] 2024-11-21T11:14:40,675 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T11:14:40,675 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T11:14:40,675 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T11:14:40,675 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T11:14:40,675 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T11:14:40,676 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:40,676 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T11:14:40,676 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T11:14:40,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T11:14:40,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T11:14:40,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:40,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:40,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T11:14:40,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T11:14:40,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:40,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:40,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T11:14:40,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T11:14:40,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:40,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:40,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T11:14:40,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T11:14:40,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:40,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T11:14:40,685 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T11:14:40,686 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740 2024-11-21T11:14:40,688 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740 2024-11-21T11:14:40,690 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T11:14:40,710 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T11:14:40,715 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-21T11:14:40,722 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T11:14:40,722 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T11:14:40,724 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-21T11:14:40,724 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-21T11:14:40,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T11:14:40,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T11:14:40,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T11:14:40,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T11:14:40,792 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/info/3102bafa902546a4b29e00718e6020ae is 138, key is test,bbb,1732187674423.fab4371e299cd292e600af4717485beb./info:regioninfo/1732187676223/Put/seqid=0 2024-11-21T11:14:40,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741866_1042 (size=36382) 2024-11-21T11:14:40,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741866_1042 (size=36382) 2024-11-21T11:14:40,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741866_1042 (size=36382) 2024-11-21T11:14:40,807 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/info/3102bafa902546a4b29e00718e6020ae 2024-11-21T11:14:40,877 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/ns/b8c70fc8e6104b80a024b35a62d69731 is 43, key is default/ns:d/1732187674031/Put/seqid=0 2024-11-21T11:14:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741867_1043 (size=5153) 2024-11-21T11:14:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741867_1043 (size=5153) 2024-11-21T11:14:40,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741867_1043 (size=5153) 2024-11-21T11:14:40,886 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/ns/b8c70fc8e6104b80a024b35a62d69731 2024-11-21T11:14:40,911 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/table/e26b6ace8fa049cdb56280507755be8d is 40, key is test/table:state/1732187676745/Put/seqid=0 2024-11-21T11:14:40,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741868_1044 (size=5165) 2024-11-21T11:14:40,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741868_1044 (size=5165) 2024-11-21T11:14:40,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741868_1044 (size=5165) 2024-11-21T11:14:40,921 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/table/e26b6ace8fa049cdb56280507755be8d 2024-11-21T11:14:40,931 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/info/3102bafa902546a4b29e00718e6020ae as hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/info/3102bafa902546a4b29e00718e6020ae 2024-11-21T11:14:40,940 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/info/3102bafa902546a4b29e00718e6020ae, entries=273, sequenceid=64, filesize=35.5 K 2024-11-21T11:14:40,943 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/ns/b8c70fc8e6104b80a024b35a62d69731 as hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/ns/b8c70fc8e6104b80a024b35a62d69731 2024-11-21T11:14:40,952 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/ns/b8c70fc8e6104b80a024b35a62d69731, entries=2, sequenceid=64, filesize=5.0 K 2024-11-21T11:14:40,954 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/table/e26b6ace8fa049cdb56280507755be8d as hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/table/e26b6ace8fa049cdb56280507755be8d 2024-11-21T11:14:40,963 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/table/e26b6ace8fa049cdb56280507755be8d, entries=2, sequenceid=64, filesize=5.0 K 2024-11-21T11:14:40,967 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 252ms, sequenceid=64, compaction requested=false; wal=null 2024-11-21T11:14:40,968 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-21T11:14:40,970 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T11:14:40,970 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T11:14:40,970 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T11:14:40,972 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T11:14:40,974 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-21T11:14:40,975 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71048386, jitterRate=0.058703452348709106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T11:14:40,975 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T11:14:40,975 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732187680676Writing region info on filesystem at 1732187680676Initializing all the Stores at 1732187680677 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187680677Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187680678 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187680678Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732187680678Obtaining lock to block concurrent updates at 1732187680715 (+37 ms)Preparing flush snapshotting stores in 1588230740 at 1732187680715Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732187680719 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732187680720 (+1 ms)Flushing 1588230740/info: creating writer at 1732187680722 (+2 ms)Flushing 1588230740/info: appending metadata at 1732187680782 (+60 ms)Flushing 1588230740/info: closing flushed file at 1732187680785 (+3 ms)Flushing 1588230740/ns: creating writer at 1732187680860 (+75 ms)Flushing 1588230740/ns: appending metadata at 1732187680876 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732187680876Flushing 1588230740/table: creating writer at 1732187680894 (+18 ms)Flushing 1588230740/table: appending metadata at 1732187680910 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732187680910Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30ac8b86: reopening flushed file at 1732187680930 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23435be4: reopening flushed file at 1732187680942 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33cf8a45: reopening flushed file at 1732187680952 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 252ms, sequenceid=64, compaction requested=false; wal=null at 1732187680967 (+15 ms)Cleaning up temporary data from old regions at 1732187680970 (+3 ms)Running coprocessor post-open hooks at 1732187680975 (+5 ms)Region opened successfully at 1732187680975 2024-11-21T11:14:40,977 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732187680645 2024-11-21T11:14:40,980 DEBUG [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T11:14:40,980 INFO [RS_OPEN_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T11:14:40,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:40,981 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:40,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:40,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:40,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:40,983 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:40,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:40,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:40,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:40,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-21T11:14:40,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 in 491 msec 2024-11-21T11:14:40,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-21T11:14:40,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 653 msec 2024-11-21T11:14:40,989 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 7b462513bfc2,45537,1732187671328 had 9 regions 2024-11-21T11:14:40,990 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true, isMeta: false 2024-11-21T11:14:40,992 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7b462513bfc2,45537,1732187671328 WAL count=1, meta=false 2024-11-21T11:14:40,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200}] 2024-11-21T11:14:40,994 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:40,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:41,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44785 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-21T11:14:41,165 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200, size=2.7 K (2774bytes) 2024-11-21T11:14:41,165 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 2024-11-21T11:14:41,166 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 after 1ms 2024-11-21T11:14:41,169 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 took 4ms 2024-11-21T11:14:41,172 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for eb15d5d5ec6aa108fb3f69b646d5757d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,173 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f1d6b26cdbeafd1feed231ef1797ec16: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 9c3ba6d97819a96bf36b5d4ae2cc3573: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,176 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c6f9184454915671110d8238a3b71824: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,178 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 05dba68ed06ee6ea199d777ec417cb17: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,179 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 78487f45e051e2d9f1275ac2ef77493d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,180 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for aa29bf2ce1b3154b66b07cc896f4188a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,182 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 42b6f7e93058e77d80053f0460a70897: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,183 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 6f3c54f1f14d06f35d9d402ce20065f9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T11:14:41,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 so closing down 2024-11-21T11:14:41,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T11:14:41,184 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T11:14:41,184 INFO [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 15 ms; skipped=10; WAL=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-21T11:14:41,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200, journal: Splitting hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200, size=2.7 K (2774bytes) at 1732187681165Finishing writing output for hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 so closing down at 1732187681184 (+19 ms)3 split writer threads finished at 1732187681184Processed 10 edits across 0 Regions in 15 ms; skipped=10; WAL=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1732187681184 2024-11-21T11:14:41,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 2024-11-21T11:14:41,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7b462513bfc2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-21T11:14:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38843 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-21T11:14:41,189 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting/7b462513bfc2%2C45537%2C1732187671328.1732187673200 to hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs 2024-11-21T11:14:41,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-21T11:14:41,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 in 194 msec 2024-11-21T11:14:41,193 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-21T11:14:41,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 in 201 msec 2024-11-21T11:14:41,197 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/WALs/7b462513bfc2,45537,1732187671328-splitting dir is empty, no logs to split. 2024-11-21T11:14:41,197 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7b462513bfc2,45537,1732187671328 WAL count=0, meta=false 2024-11-21T11:14:41,197 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 7b462513bfc2,45537,1732187671328 WAL splitting is done? wals=0, meta=false 2024-11-21T11:14:41,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN}] 2024-11-21T11:14:41,201 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,202 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,202 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,203 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,204 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,205 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,206 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,206 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,207 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,208 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,209 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,210 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,211 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,211 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,212 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,213 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T11:14:41,352 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(204): Hosts are {7b462513bfc2=0} racks are {/default-rack=0} 2024-11-21T11:14:41,353 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T11:14:41,353 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T11:14:41,353 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T11:14:41,353 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T11:14:41,353 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T11:14:41,353 INFO [7b462513bfc2:38843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T11:14:41,353 DEBUG [7b462513bfc2:38843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T11:14:41,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,354 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,355 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7b462513bfc2:45537 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:41,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=c6f9184454915671110d8238a3b71824, regionState=OPENING, regionLocation=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, regionLocation=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T11:14:41,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 from cache 2024-11-21T11:14:41,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 7b462513bfc2:45537 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,360 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7b462513bfc2:45537 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T11:14:41,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,45537,1732187671328, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=7b462513bfc2:45537 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 7b462513bfc2:45537 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b462513bfc2/172.17.0.2:45537 2024-11-21T11:14:41,463 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T11:14:41,463 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b462513bfc2,44785,1732187671410, seqNum=-1] 2024-11-21T11:14:41,464 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T11:14:41,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51171, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T11:14:41,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN because future has completed 2024-11-21T11:14:41,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:41,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN because future has completed 2024-11-21T11:14:41,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN because future has completed 2024-11-21T11:14:41,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN because future has completed 2024-11-21T11:14:41,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:41,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN because future has completed 2024-11-21T11:14:41,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN because future has completed 2024-11-21T11:14:41,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN because future has completed 2024-11-21T11:14:41,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN because future has completed 2024-11-21T11:14:41,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:41,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:41,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:41,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172}] 2024-11-21T11:14:41,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:41,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,44785,1732187671410}] 2024-11-21T11:14:41,625 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:41,626 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f3c54f1f14d06f35d9d402ce20065f9, NAME => 'test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-21T11:14:41,626 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,626 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,626 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,626 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,628 INFO [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,629 INFO [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3c54f1f14d06f35d9d402ce20065f9 columnFamilyName family 2024-11-21T11:14:41,629 DEBUG [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,630 INFO [StoreOpener-6f3c54f1f14d06f35d9d402ce20065f9-1 {}] regionserver.HStore(327): Store=6f3c54f1f14d06f35d9d402ce20065f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,630 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,631 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,633 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c3ba6d97819a96bf36b5d4ae2cc3573, NAME => 'test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-21T11:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,633 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,634 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,634 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,634 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,634 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,634 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,635 INFO [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,637 INFO [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c3ba6d97819a96bf36b5d4ae2cc3573 columnFamilyName family 2024-11-21T11:14:41,637 DEBUG [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,637 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,638 INFO [StoreOpener-9c3ba6d97819a96bf36b5d4ae2cc3573-1 {}] regionserver.HStore(327): Store=9c3ba6d97819a96bf36b5d4ae2cc3573/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,638 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,638 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 6f3c54f1f14d06f35d9d402ce20065f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69878982, jitterRate=0.04127797484397888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,638 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:41,638 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 6f3c54f1f14d06f35d9d402ce20065f9: Running coprocessor pre-open hook at 1732187681626Writing region info on filesystem at 1732187681626Initializing all the Stores at 1732187681627 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681628 (+1 ms)Cleaning up temporary data from old regions at 1732187681634 (+6 ms)Running coprocessor post-open hooks at 1732187681638 (+4 ms)Region opened successfully at 1732187681638 2024-11-21T11:14:41,639 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,640 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9., pid=80, masterSystemTime=1732187681621 2024-11-21T11:14:41,640 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,641 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,641 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,642 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:41,642 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:41,643 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:41,643 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 05dba68ed06ee6ea199d777ec417cb17, NAME => 'test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-21T11:14:41,643 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,643 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,644 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,644 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,644 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,645 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 9c3ba6d97819a96bf36b5d4ae2cc3573; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70617820, jitterRate=0.052287518978118896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,645 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:41,645 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 9c3ba6d97819a96bf36b5d4ae2cc3573: Running coprocessor pre-open hook at 1732187681634Writing region info on filesystem at 1732187681634Initializing all the Stores at 1732187681635 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681635Cleaning up temporary data from old regions at 1732187681641 (+6 ms)Running coprocessor post-open hooks at 1732187681645 (+4 ms)Region opened successfully at 1732187681645 2024-11-21T11:14:41,645 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-21T11:14:41,645 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,646 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,648 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573., pid=78, masterSystemTime=1732187681628 2024-11-21T11:14:41,649 INFO [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,650 INFO [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05dba68ed06ee6ea199d777ec417cb17 columnFamilyName family 2024-11-21T11:14:41,650 DEBUG [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,651 INFO [StoreOpener-05dba68ed06ee6ea199d777ec417cb17-1 {}] regionserver.HStore(327): Store=05dba68ed06ee6ea199d777ec417cb17/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,651 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,653 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,654 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,655 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,655 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,657 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:41,657 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:41,657 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:41,657 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,657 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 78487f45e051e2d9f1275ac2ef77493d, NAME => 'test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-21T11:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,658 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,659 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 05dba68ed06ee6ea199d777ec417cb17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62406607, jitterRate=-0.0700690895318985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,659 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:41,659 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 05dba68ed06ee6ea199d777ec417cb17: Running coprocessor pre-open hook at 1732187681644Writing region info on filesystem at 1732187681644Initializing all the Stores at 1732187681645 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681645Cleaning up temporary data from old regions at 1732187681655 (+10 ms)Running coprocessor post-open hooks at 1732187681659 (+4 ms)Region opened successfully at 1732187681659 2024-11-21T11:14:41,660 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17., pid=75, masterSystemTime=1732187681621 2024-11-21T11:14:41,660 INFO [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,662 INFO [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78487f45e051e2d9f1275ac2ef77493d columnFamilyName family 2024-11-21T11:14:41,662 DEBUG [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,663 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:41,663 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:41,663 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:41,663 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => aa29bf2ce1b3154b66b07cc896f4188a, NAME => 'test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T11:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,662 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-21T11:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,664 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,665 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,665 INFO [StoreOpener-78487f45e051e2d9f1275ac2ef77493d-1 {}] regionserver.HStore(327): Store=78487f45e051e2d9f1275ac2ef77493d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,665 INFO [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,666 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,666 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,667 INFO [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa29bf2ce1b3154b66b07cc896f4188a columnFamilyName family 2024-11-21T11:14:41,667 DEBUG [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,668 INFO [StoreOpener-aa29bf2ce1b3154b66b07cc896f4188a-1 {}] regionserver.HStore(327): Store=aa29bf2ce1b3154b66b07cc896f4188a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,668 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,668 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,668 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,668 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,669 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,670 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,671 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,671 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,671 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,672 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 78487f45e051e2d9f1275ac2ef77493d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67532798, jitterRate=0.006317108869552612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,673 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:41,673 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 78487f45e051e2d9f1275ac2ef77493d: Running coprocessor pre-open hook at 1732187681658Writing region info on filesystem at 1732187681658Initializing all the Stores at 1732187681660 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681660Cleaning up temporary data from old regions at 1732187681668 (+8 ms)Running coprocessor post-open hooks at 1732187681673 (+5 ms)Region opened successfully at 1732187681673 2024-11-21T11:14:41,673 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,673 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d., pid=81, masterSystemTime=1732187681628 2024-11-21T11:14:41,675 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened aa29bf2ce1b3154b66b07cc896f4188a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66134860, jitterRate=-0.014513790607452393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,675 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:41,675 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for aa29bf2ce1b3154b66b07cc896f4188a: Running coprocessor pre-open hook at 1732187681664Writing region info on filesystem at 1732187681664Initializing all the Stores at 1732187681665 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681665Cleaning up temporary data from old regions at 1732187681671 (+6 ms)Running coprocessor post-open hooks at 1732187681675 (+4 ms)Region opened successfully at 1732187681675 2024-11-21T11:14:41,676 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a., pid=76, masterSystemTime=1732187681621 2024-11-21T11:14:41,676 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:41,676 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:41,676 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:41,676 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 42b6f7e93058e77d80053f0460a70897, NAME => 'test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-21T11:14:41,677 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,677 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,677 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,677 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,678 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:41,678 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:41,678 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:41,678 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => c6f9184454915671110d8238a3b71824, NAME => 'test,eee,1732187674423.c6f9184454915671110d8238a3b71824.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-21T11:14:41,678 INFO [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,678 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,679 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,eee,1732187674423.c6f9184454915671110d8238a3b71824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,679 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,679 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,680 INFO [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b6f7e93058e77d80053f0460a70897 columnFamilyName family 2024-11-21T11:14:41,680 DEBUG [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,680 INFO [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,680 INFO [StoreOpener-42b6f7e93058e77d80053f0460a70897-1 {}] regionserver.HStore(327): Store=42b6f7e93058e77d80053f0460a70897/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,680 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,681 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,681 INFO [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6f9184454915671110d8238a3b71824 columnFamilyName family 2024-11-21T11:14:41,681 DEBUG [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,682 INFO [StoreOpener-c6f9184454915671110d8238a3b71824-1 {}] regionserver.HStore(327): Store=c6f9184454915671110d8238a3b71824/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,682 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,682 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,682 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,683 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,683 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,684 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,684 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,684 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,685 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,686 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 42b6f7e93058e77d80053f0460a70897; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64933961, jitterRate=-0.03240858018398285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,686 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:41,686 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 42b6f7e93058e77d80053f0460a70897: Running coprocessor pre-open hook at 1732187681677Writing region info on filesystem at 1732187681677Initializing all the Stores at 1732187681678 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681678Cleaning up temporary data from old regions at 1732187681683 (+5 ms)Running coprocessor post-open hooks at 1732187681686 (+3 ms)Region opened successfully at 1732187681686 2024-11-21T11:14:41,686 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,687 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897., pid=82, masterSystemTime=1732187681628 2024-11-21T11:14:41,688 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened c6f9184454915671110d8238a3b71824; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62343448, jitterRate=-0.07101023197174072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,688 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:41,688 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for c6f9184454915671110d8238a3b71824: Running coprocessor pre-open hook at 1732187681679Writing region info on filesystem at 1732187681679Initializing all the Stores at 1732187681680 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681680Cleaning up temporary data from old regions at 1732187681684 (+4 ms)Running coprocessor post-open hooks at 1732187681688 (+4 ms)Region opened successfully at 1732187681688 2024-11-21T11:14:41,689 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732187674423.c6f9184454915671110d8238a3b71824., pid=79, masterSystemTime=1732187681621 2024-11-21T11:14:41,689 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:41,689 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:41,689 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:41,689 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => f1d6b26cdbeafd1feed231ef1797ec16, NAME => 'test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-21T11:14:41,690 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,690 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T11:14:41,690 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,690 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,690 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:41,691 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:41,691 INFO [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,692 INFO [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1d6b26cdbeafd1feed231ef1797ec16 columnFamilyName family 2024-11-21T11:14:41,692 DEBUG [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T11:14:41,693 INFO [StoreOpener-f1d6b26cdbeafd1feed231ef1797ec16-1 {}] regionserver.HStore(327): Store=f1d6b26cdbeafd1feed231ef1797ec16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T11:14:41,693 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,694 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,695 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,695 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,695 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,697 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,698 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened f1d6b26cdbeafd1feed231ef1797ec16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59380595, jitterRate=-0.11516018211841583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T11:14:41,698 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:41,698 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for f1d6b26cdbeafd1feed231ef1797ec16: Running coprocessor pre-open hook at 1732187681690Writing region info on filesystem at 1732187681690Initializing all the Stores at 1732187681691 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732187681691Cleaning up temporary data from old regions at 1732187681695 (+4 ms)Running coprocessor post-open hooks at 1732187681698 (+3 ms)Region opened successfully at 1732187681698 2024-11-21T11:14:41,699 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16., pid=77, masterSystemTime=1732187681628 2024-11-21T11:14:41,701 DEBUG [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:41,701 INFO [RS_OPEN_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:41,708 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,708 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,709 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,709 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,709 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,710 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,710 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,723 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,724 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b6f7e93058e77d80053f0460a70897, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-21T11:14:41,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,727 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,727 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,730 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711731 2024-11-21T11:14:41,731 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,731 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,735 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,770 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,770 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741770 2024-11-21T11:14:41,770 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801770 2024-11-21T11:14:41,770 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-11-21T11:14:41,770 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,770 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,770 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,771 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,771 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,771 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,771 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,771 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,772 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,779 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,779 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78487f45e051e2d9f1275ac2ef77493d, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-21T11:14:41,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,786 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,786 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,791 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,791 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,791 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711791 2024-11-21T11:14:41,791 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,791 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,793 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,794 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,794 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,795 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,804 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,804 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741804 2024-11-21T11:14:41,804 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801804 2024-11-21T11:14:41,804 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-21T11:14:41,804 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,804 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,804 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,804 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,804 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,804 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,804 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,804 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,805 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,810 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,811 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-21T11:14:41,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,820 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,820 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,824 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,824 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,824 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,824 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,825 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,825 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,825 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,825 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,825 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711825 2024-11-21T11:14:41,825 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,825 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,827 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,828 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,828 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,829 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,840 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,840 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741840 2024-11-21T11:14:41,840 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801840 2024-11-21T11:14:41,840 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-21T11:14:41,840 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,840 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,841 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,841 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,841 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,841 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,841 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,841 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,842 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,845 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,845 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,845 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,845 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6f9184454915671110d8238a3b71824, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-21T11:14:41,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,854 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,854 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,856 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,856 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,856 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,858 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711858 2024-11-21T11:14:41,858 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,858 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,860 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,861 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,861 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,861 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,868 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,868 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741868 2024-11-21T11:14:41,868 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801868 2024-11-21T11:14:41,868 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T11:14:41,868 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,869 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,869 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,869 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,869 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,869 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,869 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,877 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,877 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,877 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,877 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c3ba6d97819a96bf36b5d4ae2cc3573, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,877 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T11:14:41,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,878 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,878 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,882 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711882 2024-11-21T11:14:41,882 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,882 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,884 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,888 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,889 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,889 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,892 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,892 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741892 2024-11-21T11:14:41,892 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801892 2024-11-21T11:14:41,892 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T11:14:41,892 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,892 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,892 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,892 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,893 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,893 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,893 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,893 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,896 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1d6b26cdbeafd1feed231ef1797ec16, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T11:14:41,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,900 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,900 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,902 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,903 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711903 2024-11-21T11:14:41,903 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,903 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,908 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,908 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,908 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,917 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,917 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741917 2024-11-21T11:14:41,917 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801917 2024-11-21T11:14:41,917 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-21T11:14:41,917 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,917 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,917 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,917 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,918 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,918 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,918 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,918 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,927 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa29bf2ce1b3154b66b07cc896f4188a, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T11:14:41,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,930 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,930 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,934 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,934 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,934 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,934 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,934 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,935 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711935 2024-11-21T11:14:41,935 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,935 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,937 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,938 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,938 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,938 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,946 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,946 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741946 2024-11-21T11:14:41,946 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801946 2024-11-21T11:14:41,946 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T11:14:41,946 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,947 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,947 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,947 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,947 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,947 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,948 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,953 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05dba68ed06ee6ea199d777ec417cb17, server=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T11:14:41,958 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,958 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,958 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,961 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,962 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711962 2024-11-21T11:14:41,962 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,962 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,964 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,971 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,971 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741971 2024-11-21T11:14:41,971 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801971 2024-11-21T11:14:41,971 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T11:14:41,971 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,971 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,971 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,971 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,971 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,971 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,971 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,971 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN 2024-11-21T11:14:41,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T11:14:41,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:41,978 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:41,978 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:41,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:41,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:41,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:41,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:41,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:41,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:41,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:41,981 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:41,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:41,982 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1d6b26cdbeafd1feed231ef1797ec16, ASSIGN to state=OFFLINE, location=null, table=test, region=f1d6b26cdbeafd1feed231ef1797ec16 to restore RIT 2024-11-21T11:14:41,982 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187711982 2024-11-21T11:14:41,982 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:41,982 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,983 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:41,986 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:41,986 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,986 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:41,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:41,991 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:41,991 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187741991 2024-11-21T11:14:41,991 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187801991 2024-11-21T11:14:41,991 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T11:14:41,991 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:41,992 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:41,992 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:41,992 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:41,992 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:41,992 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:41,992 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:41,992 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:41,993 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:41,995 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:41,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:41,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:41,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:41,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:41,998 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,998 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN 2024-11-21T11:14:41,999 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T11:14:41,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,000 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,000 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,002 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,002 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:42,002 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:42,002 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:42,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:42,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:42,003 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=05dba68ed06ee6ea199d777ec417cb17, ASSIGN to state=OFFLINE, location=null, table=test, region=05dba68ed06ee6ea199d777ec417cb17 to restore RIT 2024-11-21T11:14:42,003 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712003 2024-11-21T11:14:42,003 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,003 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,004 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,005 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,005 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:42,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,005 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,011 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742011 2024-11-21T11:14:42,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802011 2024-11-21T11:14:42,011 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T11:14:42,011 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,011 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,011 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,012 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,012 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,012 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:42,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN 2024-11-21T11:14:42,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T11:14:42,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,018 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,018 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:42,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:42,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:42,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:42,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aa29bf2ce1b3154b66b07cc896f4188a, ASSIGN to state=OFFLINE, location=null, table=test, region=aa29bf2ce1b3154b66b07cc896f4188a to restore RIT 2024-11-21T11:14:42,023 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712023 2024-11-21T11:14:42,023 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,023 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,025 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,026 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:42,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,031 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,031 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742031 2024-11-21T11:14:42,031 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802031 2024-11-21T11:14:42,031 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T11:14:42,031 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,031 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,031 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,031 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,031 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,031 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,031 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,031 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN 2024-11-21T11:14:42,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T11:14:42,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,037 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,039 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,039 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:42,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:42,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:42,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c6f9184454915671110d8238a3b71824, ASSIGN to state=OFFLINE, location=null, table=test, region=c6f9184454915671110d8238a3b71824 to restore RIT 2024-11-21T11:14:42,040 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712040 2024-11-21T11:14:42,040 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,040 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,041 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,042 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,049 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742049 2024-11-21T11:14:42,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802049 2024-11-21T11:14:42,049 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T11:14:42,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,050 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,050 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,050 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,050 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,057 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN 2024-11-21T11:14:42,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T11:14:42,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,057 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,058 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,061 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,061 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:42,062 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:42,062 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42b6f7e93058e77d80053f0460a70897, ASSIGN to state=OFFLINE, location=null, table=test, region=42b6f7e93058e77d80053f0460a70897 to restore RIT 2024-11-21T11:14:42,062 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712062 2024-11-21T11:14:42,062 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,062 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,063 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,068 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,068 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,068 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,072 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,072 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742072 2024-11-21T11:14:42,072 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802072 2024-11-21T11:14:42,072 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T11:14:42,072 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,072 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,072 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,072 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,073 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,073 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,073 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,073 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,076 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,077 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,080 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,080 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN 2024-11-21T11:14:42,080 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T11:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,081 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,081 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:42,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=78487f45e051e2d9f1275ac2ef77493d, ASSIGN to state=OFFLINE, location=null, table=test, region=78487f45e051e2d9f1275ac2ef77493d to restore RIT 2024-11-21T11:14:42,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712084 2024-11-21T11:14:42,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,085 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,087 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,087 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,087 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,090 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742090 2024-11-21T11:14:42,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802090 2024-11-21T11:14:42,090 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-21T11:14:42,090 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,091 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,091 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,091 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,091 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,091 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,092 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,093 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN 2024-11-21T11:14:42,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T11:14:42,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,096 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,096 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f3c54f1f14d06f35d9d402ce20065f9, ASSIGN to state=OFFLINE, location=null, table=test, region=6f3c54f1f14d06f35d9d402ce20065f9 to restore RIT 2024-11-21T11:14:42,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712105 2024-11-21T11:14:42,105 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,105 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,106 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,108 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,108 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,108 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,112 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,112 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742112 2024-11-21T11:14:42,112 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802112 2024-11-21T11:14:42,112 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,112 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,112 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,112 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,112 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,112 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,113 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN 2024-11-21T11:14:42,117 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T11:14:42,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,118 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,118 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573, ASSIGN to state=OFFLINE, location=null, table=test, region=9c3ba6d97819a96bf36b5d4ae2cc3573 to restore RIT 2024-11-21T11:14:42,121 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712121 2024-11-21T11:14:42,121 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,121 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,122 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,123 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,128 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742128 2024-11-21T11:14:42,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802128 2024-11-21T11:14:42,128 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,128 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,128 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,128 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,129 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T11:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,133 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,133 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712136 2024-11-21T11:14:42,136 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,136 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,139 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,139 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,139 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,143 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,143 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742143 2024-11-21T11:14:42,143 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802143 2024-11-21T11:14:42,143 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,143 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,143 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,143 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,143 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,143 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,143 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,143 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,144 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.1732187673200, worker=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,148 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T11:14:42,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,148 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,148 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712152 2024-11-21T11:14:42,152 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,152 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,154 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,155 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,160 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,160 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742160 2024-11-21T11:14:42,160 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802160 2024-11-21T11:14:42,160 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T11:14:42,160 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,160 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,160 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,160 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,160 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,160 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T11:14:42,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b462513bfc2,44785,1732187671410 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,164 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,164 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,168 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,168 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,168 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712168 2024-11-21T11:14:42,169 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,169 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,170 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,173 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,173 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,173 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742175 2024-11-21T11:14:42,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802175 2024-11-21T11:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,176 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T11:14:42,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,179 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,179 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T11:14:42,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,180 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,180 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,183 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712183 2024-11-21T11:14:42,183 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,183 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,184 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,191 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,191 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742191 2024-11-21T11:14:42,191 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802191 2024-11-21T11:14:42,191 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T11:14:42,191 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,191 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,191 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,191 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,192 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,192 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T11:14:42,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,196 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T11:14:42,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,197 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,197 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,199 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712200 2024-11-21T11:14:42,200 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,200 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,201 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,202 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,206 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742206 2024-11-21T11:14:42,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802207 2024-11-21T11:14:42,207 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,207 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,207 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,207 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,207 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,207 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,207 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T11:14:42,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,210 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 7b462513bfc2%2C45537%2C1732187671328.meta.1732187673708.meta, worker=7b462513bfc2,33361,1732187671172 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T11:14:42,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T11:14:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,211 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,211 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,214 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712214 2024-11-21T11:14:42,214 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,214 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,215 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T11:14:42,216 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,221 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742221 2024-11-21T11:14:42,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802221 2024-11-21T11:14:42,221 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,221 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T11:14:42,221 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,221 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,221 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,221 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T11:14:42,222 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T11:14:42,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T11:14:42,222 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T11:14:42,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T11:14:42,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T11:14:42,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T11:14:42,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T11:14:42,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb15d5d5ec6aa108fb3f69b646d5757d, REOPEN/MOVE 2024-11-21T11:14:42,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true 2024-11-21T11:14:42,225 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T11:14:42,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T11:14:42,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=65 2024-11-21T11:14:42,225 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=7b462513bfc2,44785,1732187671410, table=hbase:meta, region=1588230740 2024-11-21T11:14:42,225 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b462513bfc2,44785,1732187671410, state=OPEN 2024-11-21T11:14:42,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T11:14:42,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T11:14:42,230 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732187712230 2024-11-21T11:14:42,230 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T11:14:42,230 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T11:14:42,231 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T11:14:42,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 6f3c54f1f14d06f35d9d402ce20065f9, server=7b462513bfc2,33361,1732187671172:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 7b462513bfc2,45537,1732187671328, splitWal=true, meta=true exec-time=2.1970 sec 2024-11-21T11:14:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f15a03020943182be6b4e66ea5cf19a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fab4371e299cd292e600af4717485beb, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78487f45e051e2d9f1275ac2ef77493d, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be5404b662521c75e774e2749bf34474, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6f9184454915671110d8238a3b71824, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76043d320d93c5a6c1b870931771d01c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=075843f8fc377ab19ddd09be30eeaa38, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42b6f7e93058e77d80053f0460a70897, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=feb12ac653343cadcf32bc8dff1d6846, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06bb2d53bc41f5595f048fbf33d2e7f6, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f3c54f1f14d06f35d9d402ce20065f9, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1d6b26cdbeafd1feed231ef1797ec16, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad417eadf023765c2d8ea0936fcb018c, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9501094853170e6297b53847d78f71c8, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb15d5d5ec6aa108fb3f69b646d5757d, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=5 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea0e500737edd661e0f9e38b0cf0f99a, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b570ae93fb4b7f17a9d44dce21589f, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa29bf2ce1b3154b66b07cc896f4188a, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f738404802b7de06817afff11fed6aa, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c3ba6d97819a96bf36b5d4ae2cc3573, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46a04157c564adf02211263396700675, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ad652628fe53a6af579af31b1d64bdf, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05dba68ed06ee6ea199d777ec417cb17, regionState=OPENING, lastHost=7b462513bfc2,45537,1732187671328, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=993840b9e61b5baf1c5a77b64bde7e9d, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f52493bc04c75885babe0eeaf6faf9b0, regionState=OPEN, lastHost=7b462513bfc2,44785,1732187671410, regionLocation=7b462513bfc2,44785,1732187671410, openSeqNum=2 2024-11-21T11:14:42,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325cd18a1a1b735062fa7f51132edef4, regionState=OPEN, lastHost=7b462513bfc2,33361,1732187671172, regionLocation=7b462513bfc2,33361,1732187671172, openSeqNum=2 2024-11-21T11:14:42,236 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T11:14:42,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732187742236 2024-11-21T11:14:42,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732187802236 2024-11-21T11:14:42,236 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T11:14:42,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=346 (was 379), OpenFileDescriptor=616 (was 637), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=573 (was 573), ProcessCount=11 (was 11), AvailableMemoryMB=5095 (was 5172) 2024-11-21T11:14:42,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T11:14:42,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:14:42,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:42,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T11:14:42,255 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T11:14:42,255 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-11-21T11:14:42,255 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7b462513bfc2,38843,1732187670157 2024-11-21T11:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T11:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:42,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:42,257 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T11:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:42,257 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:42,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,258 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b462513bfc2,33361,1732187671172' ***** 2024-11-21T11:14:42,258 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T11:14:42,258 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b462513bfc2,44785,1732187671410' ***** 2024-11-21T11:14:42,258 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T11:14:42,258 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T11:14:42,258 INFO [RS:0;7b462513bfc2:33361 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T11:14:42,258 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T11:14:42,258 INFO [RS:0;7b462513bfc2:33361 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T11:14:42,259 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T11:14:42,259 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 325cd18a1a1b735062fa7f51132edef4 2024-11-21T11:14:42,259 INFO [RS:2;7b462513bfc2:44785 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T11:14:42,259 INFO [RS:2;7b462513bfc2:44785 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T11:14:42,259 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for fab4371e299cd292e600af4717485beb 2024-11-21T11:14:42,259 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T11:14:42,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:42,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:42,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f15a03020943182be6b4e66ea5cf19a 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for ad417eadf023765c2d8ea0936fcb018c 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for f1d6b26cdbeafd1feed231ef1797ec16 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 9c3ba6d97819a96bf36b5d4ae2cc3573 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 46a04157c564adf02211263396700675 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 993840b9e61b5baf1c5a77b64bde7e9d 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 42b6f7e93058e77d80053f0460a70897 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 78487f45e051e2d9f1275ac2ef77493d 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 76043d320d93c5a6c1b870931771d01c 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for f52493bc04c75885babe0eeaf6faf9b0 2024-11-21T11:14:42,260 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for aa29bf2ce1b3154b66b07cc896f4188a 2024-11-21T11:14:42,260 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for ea0e500737edd661e0f9e38b0cf0f99a 2024-11-21T11:14:42,260 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 05dba68ed06ee6ea199d777ec417cb17 2024-11-21T11:14:42,260 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for c6f9184454915671110d8238a3b71824 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(959): stopping server 7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ad652628fe53a6af579af31b1d64bdf 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7b462513bfc2:44785. 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for be5404b662521c75e774e2749bf34474 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 075843f8fc377ab19ddd09be30eeaa38 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f738404802b7de06817afff11fed6aa 2024-11-21T11:14:42,261 DEBUG [RS:2;7b462513bfc2:44785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for d2b570ae93fb4b7f17a9d44dce21589f 2024-11-21T11:14:42,261 DEBUG [RS:2;7b462513bfc2:44785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 06bb2d53bc41f5595f048fbf33d2e7f6 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 9501094853170e6297b53847d78f71c8 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(3091): Received CLOSE for 6f3c54f1f14d06f35d9d402ce20065f9 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(959): stopping server 7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T11:14:42,261 INFO [RS:0;7b462513bfc2:33361 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7b462513bfc2:33361. 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T11:14:42,261 DEBUG [RS:0;7b462513bfc2:33361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T11:14:42,261 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T11:14:42,261 DEBUG [RS:0;7b462513bfc2:33361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,261 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 325cd18a1a1b735062fa7f51132edef4, disabling compactions & flushes 2024-11-21T11:14:42,262 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fab4371e299cd292e600af4717485beb, disabling compactions & flushes 2024-11-21T11:14:42,262 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:42,262 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. after waiting 0 ms 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. after waiting 0 ms 2024-11-21T11:14:42,262 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:42,262 DEBUG [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(1351): Waiting on 05dba68ed06ee6ea199d777ec417cb17, 06bb2d53bc41f5595f048fbf33d2e7f6, 075843f8fc377ab19ddd09be30eeaa38, 325cd18a1a1b735062fa7f51132edef4, 5ad652628fe53a6af579af31b1d64bdf, 6f3c54f1f14d06f35d9d402ce20065f9, 8f738404802b7de06817afff11fed6aa, 9501094853170e6297b53847d78f71c8, aa29bf2ce1b3154b66b07cc896f4188a, be5404b662521c75e774e2749bf34474, c6f9184454915671110d8238a3b71824, d2b570ae93fb4b7f17a9d44dce21589f, eb15d5d5ec6aa108fb3f69b646d5757d 2024-11-21T11:14:42,264 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-21T11:14:42,264 DEBUG [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3f15a03020943182be6b4e66ea5cf19a, 42b6f7e93058e77d80053f0460a70897, 46a04157c564adf02211263396700675, 76043d320d93c5a6c1b870931771d01c, 78487f45e051e2d9f1275ac2ef77493d, 993840b9e61b5baf1c5a77b64bde7e9d, 9c3ba6d97819a96bf36b5d4ae2cc3573, ad417eadf023765c2d8ea0936fcb018c, ea0e500737edd661e0f9e38b0cf0f99a, f1d6b26cdbeafd1feed231ef1797ec16, f52493bc04c75885babe0eeaf6faf9b0, fab4371e299cd292e600af4717485beb, feb12ac653343cadcf32bc8dff1d6846 2024-11-21T11:14:42,264 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T11:14:42,264 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T11:14:42,264 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T11:14:42,264 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T11:14:42,264 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T11:14:42,264 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-21T11:14:42,267 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/325cd18a1a1b735062fa7f51132edef4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,268 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/fab4371e299cd292e600af4717485beb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,268 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:42,268 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 325cd18a1a1b735062fa7f51132edef4: Waiting for close lock at 1732187682261Running coprocessor pre-close hooks at 1732187682261Disabling compacts and flushes for region at 1732187682261Disabling writes for close at 1732187682262 (+1 ms)Writing region close event to WAL at 1732187682262Running coprocessor post-close hooks at 1732187682268 (+6 ms)Closed at 1732187682268 2024-11-21T11:14:42,268 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732187674423.325cd18a1a1b735062fa7f51132edef4. 2024-11-21T11:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa29bf2ce1b3154b66b07cc896f4188a, disabling compactions & flushes 2024-11-21T11:14:42,269 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. after waiting 0 ms 2024-11-21T11:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:42,269 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:42,269 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fab4371e299cd292e600af4717485beb: Waiting for close lock at 1732187682261Running coprocessor pre-close hooks at 1732187682261Disabling compacts and flushes for region at 1732187682261Disabling writes for close at 1732187682262 (+1 ms)Writing region close event to WAL at 1732187682262Running coprocessor post-close hooks at 1732187682269 (+7 ms)Closed at 1732187682269 2024-11-21T11:14:42,270 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732187674423.fab4371e299cd292e600af4717485beb. 2024-11-21T11:14:42,270 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f15a03020943182be6b4e66ea5cf19a, disabling compactions & flushes 2024-11-21T11:14:42,270 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:42,270 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:42,270 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. after waiting 0 ms 2024-11-21T11:14:42,270 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:42,274 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/aa29bf2ce1b3154b66b07cc896f4188a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,275 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa29bf2ce1b3154b66b07cc896f4188a: Waiting for close lock at 1732187682269Running coprocessor pre-close hooks at 1732187682269Disabling compacts and flushes for region at 1732187682269Disabling writes for close at 1732187682269Writing region close event to WAL at 1732187682269Running coprocessor post-close hooks at 1732187682275 (+6 ms)Closed at 1732187682275 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732187674423.aa29bf2ce1b3154b66b07cc896f4188a. 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/3f15a03020943182be6b4e66ea5cf19a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 05dba68ed06ee6ea199d777ec417cb17, disabling compactions & flushes 2024-11-21T11:14:42,275 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. after waiting 0 ms 2024-11-21T11:14:42,275 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:42,276 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:42,276 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f15a03020943182be6b4e66ea5cf19a: Waiting for close lock at 1732187682270Running coprocessor pre-close hooks at 1732187682270Disabling compacts and flushes for region at 1732187682270Disabling writes for close at 1732187682270Writing region close event to WAL at 1732187682270Running coprocessor post-close hooks at 1732187682276 (+6 ms)Closed at 1732187682276 2024-11-21T11:14:42,276 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732187674423.3f15a03020943182be6b4e66ea5cf19a. 2024-11-21T11:14:42,276 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad417eadf023765c2d8ea0936fcb018c, disabling compactions & flushes 2024-11-21T11:14:42,276 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:42,276 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:42,276 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. after waiting 0 ms 2024-11-21T11:14:42,276 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:42,278 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/info/997038ba3deb4d57bceead54cee56bb1 is 138, key is test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d./info:regioninfo/1732187681358/Put/seqid=0 2024-11-21T11:14:42,279 INFO [regionserver/7b462513bfc2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:14:42,281 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/05dba68ed06ee6ea199d777ec417cb17/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,282 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:42,282 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 05dba68ed06ee6ea199d777ec417cb17: Waiting for close lock at 1732187682275Running coprocessor pre-close hooks at 1732187682275Disabling compacts and flushes for region at 1732187682275Disabling writes for close at 1732187682275Writing region close event to WAL at 1732187682276 (+1 ms)Running coprocessor post-close hooks at 1732187682282 (+6 ms)Closed at 1732187682282 2024-11-21T11:14:42,282 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732187674423.05dba68ed06ee6ea199d777ec417cb17. 2024-11-21T11:14:42,282 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c6f9184454915671110d8238a3b71824, disabling compactions & flushes 2024-11-21T11:14:42,282 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:42,282 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:42,282 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732187674423.c6f9184454915671110d8238a3b71824. after waiting 0 ms 2024-11-21T11:14:42,282 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:42,293 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ad417eadf023765c2d8ea0936fcb018c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741869_1045 (size=8195) 2024-11-21T11:14:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741869_1045 (size=8195) 2024-11-21T11:14:42,294 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:42,294 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad417eadf023765c2d8ea0936fcb018c: Waiting for close lock at 1732187682276Running coprocessor pre-close hooks at 1732187682276Disabling compacts and flushes for region at 1732187682276Disabling writes for close at 1732187682276Writing region close event to WAL at 1732187682280 (+4 ms)Running coprocessor post-close hooks at 1732187682294 (+14 ms)Closed at 1732187682294 2024-11-21T11:14:42,294 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732187674423.ad417eadf023765c2d8ea0936fcb018c. 2024-11-21T11:14:42,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741869_1045 (size=8195) 2024-11-21T11:14:42,294 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing feb12ac653343cadcf32bc8dff1d6846, disabling compactions & flushes 2024-11-21T11:14:42,295 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:42,295 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:42,295 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. after waiting 0 ms 2024-11-21T11:14:42,295 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:42,295 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/info/997038ba3deb4d57bceead54cee56bb1 2024-11-21T11:14:42,302 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/.tmp/info/997038ba3deb4d57bceead54cee56bb1 as hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/info/997038ba3deb4d57bceead54cee56bb1 2024-11-21T11:14:42,310 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/info/997038ba3deb4d57bceead54cee56bb1, entries=24, sequenceid=75, filesize=8.0 K 2024-11-21T11:14:42,311 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=75, compaction requested=false 2024-11-21T11:14:42,312 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/c6f9184454915671110d8238a3b71824/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,313 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:42,313 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c6f9184454915671110d8238a3b71824: Waiting for close lock at 1732187682282Running coprocessor pre-close hooks at 1732187682282Disabling compacts and flushes for region at 1732187682282Disabling writes for close at 1732187682282Writing region close event to WAL at 1732187682283 (+1 ms)Running coprocessor post-close hooks at 1732187682313 (+30 ms)Closed at 1732187682313 2024-11-21T11:14:42,314 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732187674423.c6f9184454915671110d8238a3b71824. 2024-11-21T11:14:42,314 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ad652628fe53a6af579af31b1d64bdf, disabling compactions & flushes 2024-11-21T11:14:42,314 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:42,314 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:42,314 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/feb12ac653343cadcf32bc8dff1d6846/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,314 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. after waiting 0 ms 2024-11-21T11:14:42,314 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:42,315 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:42,315 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for feb12ac653343cadcf32bc8dff1d6846: Waiting for close lock at 1732187682294Running coprocessor pre-close hooks at 1732187682294Disabling compacts and flushes for region at 1732187682294Disabling writes for close at 1732187682295 (+1 ms)Writing region close event to WAL at 1732187682309 (+14 ms)Running coprocessor post-close hooks at 1732187682315 (+6 ms)Closed at 1732187682315 2024-11-21T11:14:42,315 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732187674423.feb12ac653343cadcf32bc8dff1d6846. 2024-11-21T11:14:42,315 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1d6b26cdbeafd1feed231ef1797ec16, disabling compactions & flushes 2024-11-21T11:14:42,315 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:42,315 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:42,315 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. after waiting 0 ms 2024-11-21T11:14:42,315 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:42,319 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-21T11:14:42,319 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T11:14:42,320 INFO [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T11:14:42,320 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732187682264Running coprocessor pre-close hooks at 1732187682264Disabling compacts and flushes for region at 1732187682264Disabling writes for close at 1732187682264Obtaining lock to block concurrent updates at 1732187682264Preparing flush snapshotting stores in 1588230740 at 1732187682264Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732187682271 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732187682272 (+1 ms)Flushing 1588230740/info: creating writer at 1732187682272Flushing 1588230740/info: appending metadata at 1732187682278 (+6 ms)Flushing 1588230740/info: closing flushed file at 1732187682278Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e69df54: reopening flushed file at 1732187682302 (+24 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=75, compaction requested=false at 1732187682311 (+9 ms)Writing region close event to WAL at 1732187682314 (+3 ms)Running coprocessor post-close hooks at 1732187682319 (+5 ms)Closed at 1732187682320 (+1 ms) 2024-11-21T11:14:42,320 DEBUG [RS_CLOSE_META-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T11:14:42,320 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f1d6b26cdbeafd1feed231ef1797ec16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,320 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/5ad652628fe53a6af579af31b1d64bdf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,321 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:42,321 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1d6b26cdbeafd1feed231ef1797ec16: Waiting for close lock at 1732187682315Running coprocessor pre-close hooks at 1732187682315Disabling compacts and flushes for region at 1732187682315Disabling writes for close at 1732187682315Writing region close event to WAL at 1732187682316 (+1 ms)Running coprocessor post-close hooks at 1732187682321 (+5 ms)Closed at 1732187682321 2024-11-21T11:14:42,321 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732187674423.f1d6b26cdbeafd1feed231ef1797ec16. 2024-11-21T11:14:42,321 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ad652628fe53a6af579af31b1d64bdf: Waiting for close lock at 1732187682314Running coprocessor pre-close hooks at 1732187682314Disabling compacts and flushes for region at 1732187682314Disabling writes for close at 1732187682314Writing region close event to WAL at 1732187682315 (+1 ms)Running coprocessor post-close hooks at 1732187682321 (+6 ms)Closed at 1732187682321 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9c3ba6d97819a96bf36b5d4ae2cc3573, disabling compactions & flushes 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732187674423.5ad652628fe53a6af579af31b1d64bdf. 2024-11-21T11:14:42,322 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. after waiting 0 ms 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb15d5d5ec6aa108fb3f69b646d5757d, disabling compactions & flushes 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:42,322 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. after waiting 0 ms 2024-11-21T11:14:42,322 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:42,326 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/eb15d5d5ec6aa108fb3f69b646d5757d/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-21T11:14:42,326 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9c3ba6d97819a96bf36b5d4ae2cc3573/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,328 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9c3ba6d97819a96bf36b5d4ae2cc3573: Waiting for close lock at 1732187682322Running coprocessor pre-close hooks at 1732187682322Disabling compacts and flushes for region at 1732187682322Disabling writes for close at 1732187682322Writing region close event to WAL at 1732187682322Running coprocessor post-close hooks at 1732187682328 (+6 ms)Closed at 1732187682328 2024-11-21T11:14:42,328 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb15d5d5ec6aa108fb3f69b646d5757d: Waiting for close lock at 1732187682322Running coprocessor pre-close hooks at 1732187682322Disabling compacts and flushes for region at 1732187682322Disabling writes for close at 1732187682322Writing region close event to WAL at 1732187682322Running coprocessor post-close hooks at 1732187682328 (+6 ms)Closed at 1732187682328 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732187674423.9c3ba6d97819a96bf36b5d4ae2cc3573. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732187674423.eb15d5d5ec6aa108fb3f69b646d5757d. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be5404b662521c75e774e2749bf34474, disabling compactions & flushes 2024-11-21T11:14:42,328 INFO [regionserver/7b462513bfc2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:14:42,328 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46a04157c564adf02211263396700675, disabling compactions & flushes 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:42,328 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. after waiting 0 ms 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:42,328 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:42,329 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732187674423.46a04157c564adf02211263396700675. after waiting 0 ms 2024-11-21T11:14:42,329 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:42,333 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/46a04157c564adf02211263396700675/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,333 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/be5404b662521c75e774e2749bf34474/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,334 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:42,334 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be5404b662521c75e774e2749bf34474: Waiting for close lock at 1732187682328Running coprocessor pre-close hooks at 1732187682328Disabling compacts and flushes for region at 1732187682328Disabling writes for close at 1732187682328Writing region close event to WAL at 1732187682329 (+1 ms)Running coprocessor post-close hooks at 1732187682334 (+5 ms)Closed at 1732187682334 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46a04157c564adf02211263396700675: Waiting for close lock at 1732187682328Running coprocessor pre-close hooks at 1732187682328Disabling compacts and flushes for region at 1732187682328Disabling writes for close at 1732187682329 (+1 ms)Writing region close event to WAL at 1732187682329Running coprocessor post-close hooks at 1732187682334 (+5 ms)Closed at 1732187682334 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732187674423.be5404b662521c75e774e2749bf34474. 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732187674423.46a04157c564adf02211263396700675. 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 075843f8fc377ab19ddd09be30eeaa38, disabling compactions & flushes 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 993840b9e61b5baf1c5a77b64bde7e9d, disabling compactions & flushes 2024-11-21T11:14:42,334 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:42,334 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:42,334 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:42,335 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. after waiting 0 ms 2024-11-21T11:14:42,335 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. after waiting 0 ms 2024-11-21T11:14:42,335 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:42,335 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:42,339 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/993840b9e61b5baf1c5a77b64bde7e9d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,339 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/075843f8fc377ab19ddd09be30eeaa38/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,339 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:42,339 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 993840b9e61b5baf1c5a77b64bde7e9d: Waiting for close lock at 1732187682334Running coprocessor pre-close hooks at 1732187682334Disabling compacts and flushes for region at 1732187682334Disabling writes for close at 1732187682335 (+1 ms)Writing region close event to WAL at 1732187682335Running coprocessor post-close hooks at 1732187682339 (+4 ms)Closed at 1732187682339 2024-11-21T11:14:42,339 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 075843f8fc377ab19ddd09be30eeaa38: Waiting for close lock at 1732187682334Running coprocessor pre-close hooks at 1732187682334Disabling compacts and flushes for region at 1732187682334Disabling writes for close at 1732187682335 (+1 ms)Writing region close event to WAL at 1732187682335Running coprocessor post-close hooks at 1732187682339 (+4 ms)Closed at 1732187682339 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732187674423.993840b9e61b5baf1c5a77b64bde7e9d. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732187674423.075843f8fc377ab19ddd09be30eeaa38. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f738404802b7de06817afff11fed6aa, disabling compactions & flushes 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 42b6f7e93058e77d80053f0460a70897, disabling compactions & flushes 2024-11-21T11:14:42,340 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:42,340 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. after waiting 0 ms 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. after waiting 0 ms 2024-11-21T11:14:42,340 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:42,344 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/8f738404802b7de06817afff11fed6aa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,345 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/42b6f7e93058e77d80053f0460a70897/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,345 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:42,345 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f738404802b7de06817afff11fed6aa: Waiting for close lock at 1732187682340Running coprocessor pre-close hooks at 1732187682340Disabling compacts and flushes for region at 1732187682340Disabling writes for close at 1732187682340Writing region close event to WAL at 1732187682340Running coprocessor post-close hooks at 1732187682345 (+5 ms)Closed at 1732187682345 2024-11-21T11:14:42,345 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732187674423.8f738404802b7de06817afff11fed6aa. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 42b6f7e93058e77d80053f0460a70897: Waiting for close lock at 1732187682340Running coprocessor pre-close hooks at 1732187682340Disabling compacts and flushes for region at 1732187682340Disabling writes for close at 1732187682340Writing region close event to WAL at 1732187682341 (+1 ms)Running coprocessor post-close hooks at 1732187682345 (+4 ms)Closed at 1732187682345 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2b570ae93fb4b7f17a9d44dce21589f, disabling compactions & flushes 2024-11-21T11:14:42,346 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732187674423.42b6f7e93058e77d80053f0460a70897. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. after waiting 0 ms 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 78487f45e051e2d9f1275ac2ef77493d, disabling compactions & flushes 2024-11-21T11:14:42,346 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. after waiting 0 ms 2024-11-21T11:14:42,346 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:42,350 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/d2b570ae93fb4b7f17a9d44dce21589f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,351 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:42,351 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2b570ae93fb4b7f17a9d44dce21589f: Waiting for close lock at 1732187682346Running coprocessor pre-close hooks at 1732187682346Disabling compacts and flushes for region at 1732187682346Disabling writes for close at 1732187682346Writing region close event to WAL at 1732187682346Running coprocessor post-close hooks at 1732187682351 (+5 ms)Closed at 1732187682351 2024-11-21T11:14:42,351 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/78487f45e051e2d9f1275ac2ef77493d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,351 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732187674423.d2b570ae93fb4b7f17a9d44dce21589f. 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06bb2d53bc41f5595f048fbf33d2e7f6, disabling compactions & flushes 2024-11-21T11:14:42,352 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. after waiting 0 ms 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:42,352 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 78487f45e051e2d9f1275ac2ef77493d: Waiting for close lock at 1732187682346Running coprocessor pre-close hooks at 1732187682346Disabling compacts and flushes for region at 1732187682346Disabling writes for close at 1732187682346Writing region close event to WAL at 1732187682347 (+1 ms)Running coprocessor post-close hooks at 1732187682352 (+5 ms)Closed at 1732187682352 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732187674423.78487f45e051e2d9f1275ac2ef77493d. 2024-11-21T11:14:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 76043d320d93c5a6c1b870931771d01c, disabling compactions & flushes 2024-11-21T11:14:42,353 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:42,353 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:42,353 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. after waiting 0 ms 2024-11-21T11:14:42,353 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:42,357 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/06bb2d53bc41f5595f048fbf33d2e7f6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,357 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/76043d320d93c5a6c1b870931771d01c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,357 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06bb2d53bc41f5595f048fbf33d2e7f6: Waiting for close lock at 1732187682351Running coprocessor pre-close hooks at 1732187682351Disabling compacts and flushes for region at 1732187682351Disabling writes for close at 1732187682352 (+1 ms)Writing region close event to WAL at 1732187682352Running coprocessor post-close hooks at 1732187682357 (+5 ms)Closed at 1732187682357 2024-11-21T11:14:42,358 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 76043d320d93c5a6c1b870931771d01c: Waiting for close lock at 1732187682352Running coprocessor pre-close hooks at 1732187682352Disabling compacts and flushes for region at 1732187682352Disabling writes for close at 1732187682353 (+1 ms)Writing region close event to WAL at 1732187682353Running coprocessor post-close hooks at 1732187682358 (+5 ms)Closed at 1732187682358 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732187674423.06bb2d53bc41f5595f048fbf33d2e7f6. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732187674423.76043d320d93c5a6c1b870931771d01c. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9501094853170e6297b53847d78f71c8, disabling compactions & flushes 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f52493bc04c75885babe0eeaf6faf9b0, disabling compactions & flushes 2024-11-21T11:14:42,358 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:42,358 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. after waiting 0 ms 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. after waiting 0 ms 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:42,363 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/9501094853170e6297b53847d78f71c8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,363 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/f52493bc04c75885babe0eeaf6faf9b0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,364 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:42,364 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f52493bc04c75885babe0eeaf6faf9b0: Waiting for close lock at 1732187682358Running coprocessor pre-close hooks at 1732187682358Disabling compacts and flushes for region at 1732187682358Disabling writes for close at 1732187682358Writing region close event to WAL at 1732187682359 (+1 ms)Running coprocessor post-close hooks at 1732187682364 (+5 ms)Closed at 1732187682364 2024-11-21T11:14:42,364 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:42,364 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732187674423.f52493bc04c75885babe0eeaf6faf9b0. 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9501094853170e6297b53847d78f71c8: Waiting for close lock at 1732187682358Running coprocessor pre-close hooks at 1732187682358Disabling compacts and flushes for region at 1732187682358Disabling writes for close at 1732187682358Writing region close event to WAL at 1732187682359 (+1 ms)Running coprocessor post-close hooks at 1732187682364 (+5 ms)Closed at 1732187682364 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ea0e500737edd661e0f9e38b0cf0f99a, disabling compactions & flushes 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732187674423.9501094853170e6297b53847d78f71c8. 2024-11-21T11:14:42,365 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. after waiting 0 ms 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6f3c54f1f14d06f35d9d402ce20065f9, disabling compactions & flushes 2024-11-21T11:14:42,365 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. after waiting 0 ms 2024-11-21T11:14:42,365 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:42,369 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/ea0e500737edd661e0f9e38b0cf0f99a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,369 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/data/default/test/6f3c54f1f14d06f35d9d402ce20065f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T11:14:42,370 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:42,370 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ea0e500737edd661e0f9e38b0cf0f99a: Waiting for close lock at 1732187682365Running coprocessor pre-close hooks at 1732187682365Disabling compacts and flushes for region at 1732187682365Disabling writes for close at 1732187682365Writing region close event to WAL at 1732187682365Running coprocessor post-close hooks at 1732187682370 (+5 ms)Closed at 1732187682370 2024-11-21T11:14:42,370 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732187674423.ea0e500737edd661e0f9e38b0cf0f99a. 2024-11-21T11:14:42,370 INFO [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:42,370 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6f3c54f1f14d06f35d9d402ce20065f9: Waiting for close lock at 1732187682365Running coprocessor pre-close hooks at 1732187682365Disabling compacts and flushes for region at 1732187682365Disabling writes for close at 1732187682365Writing region close event to WAL at 1732187682366 (+1 ms)Running coprocessor post-close hooks at 1732187682370 (+4 ms)Closed at 1732187682370 2024-11-21T11:14:42,370 DEBUG [RS_CLOSE_REGION-regionserver/7b462513bfc2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732187674423.6f3c54f1f14d06f35d9d402ce20065f9. 2024-11-21T11:14:42,463 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.HRegionServer(976): stopping server 7b462513bfc2,33361,1732187671172; all regions closed. 2024-11-21T11:14:42,464 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.HRegionServer(976): stopping server 7b462513bfc2,44785,1732187671410; all regions closed. 2024-11-21T11:14:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741833_1009 (size=7074) 2024-11-21T11:14:42,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741833_1009 (size=7074) 2024-11-21T11:14:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741865_1041 (size=4204) 2024-11-21T11:14:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741833_1009 (size=7074) 2024-11-21T11:14:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741865_1041 (size=4204) 2024-11-21T11:14:42,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741865_1041 (size=4204) 2024-11-21T11:14:42,472 DEBUG [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs 2024-11-21T11:14:42,472 DEBUG [RS:0;7b462513bfc2:33361 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs 2024-11-21T11:14:42,473 INFO [RS:0;7b462513bfc2:33361 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b462513bfc2%2C33361%2C1732187671172:(num 1732187673182) 2024-11-21T11:14:42,473 INFO [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b462513bfc2%2C44785%2C1732187671410.meta:.meta(num 1732187680654) 2024-11-21T11:14:42,473 DEBUG [RS:0;7b462513bfc2:33361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,473 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:14:42,473 INFO [RS:0;7b462513bfc2:33361 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T11:14:42,473 INFO [RS:0;7b462513bfc2:33361 {}] hbase.ChoreService(370): Chore service for: regionserver/7b462513bfc2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T11:14:42,473 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T11:14:42,474 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T11:14:42,474 INFO [regionserver/7b462513bfc2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T11:14:42,474 INFO [RS:0;7b462513bfc2:33361 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T11:14:42,474 INFO [RS:0;7b462513bfc2:33361 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T11:14:42,474 INFO [RS:0;7b462513bfc2:33361 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33361 2024-11-21T11:14:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741835_1011 (size=7064) 2024-11-21T11:14:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741835_1011 (size=7064) 2024-11-21T11:14:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741835_1011 (size=7064) 2024-11-21T11:14:42,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33361-0x1013a3c2d4c0001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b462513bfc2,33361,1732187671172 2024-11-21T11:14:42,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T11:14:42,478 INFO [RS:0;7b462513bfc2:33361 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T11:14:42,479 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b462513bfc2,33361,1732187671172] 2024-11-21T11:14:42,480 DEBUG [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/oldWALs 2024-11-21T11:14:42,481 INFO [RS:2;7b462513bfc2:44785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b462513bfc2%2C44785%2C1732187671410:(num 1732187673230) 2024-11-21T11:14:42,481 DEBUG [RS:2;7b462513bfc2:44785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T11:14:42,481 INFO [RS:2;7b462513bfc2:44785 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T11:14:42,481 INFO [RS:2;7b462513bfc2:44785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T11:14:42,481 INFO [RS:2;7b462513bfc2:44785 {}] hbase.ChoreService(370): Chore service for: regionserver/7b462513bfc2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T11:14:42,481 INFO [RS:2;7b462513bfc2:44785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T11:14:42,481 INFO [regionserver/7b462513bfc2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T11:14:42,481 INFO [RS:2;7b462513bfc2:44785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44785 2024-11-21T11:14:42,482 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b462513bfc2,33361,1732187671172 already deleted, retry=false 2024-11-21T11:14:42,483 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b462513bfc2,33361,1732187671172 expired; onlineServers=1 2024-11-21T11:14:42,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44785-0x1013a3c2d4c0003, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b462513bfc2,44785,1732187671410 2024-11-21T11:14:42,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T11:14:42,483 INFO [RS:2;7b462513bfc2:44785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T11:14:42,485 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b462513bfc2,44785,1732187671410] 2024-11-21T11:14:42,486 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b462513bfc2,44785,1732187671410 already deleted, retry=false 2024-11-21T11:14:42,486 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b462513bfc2,44785,1732187671410 expired; onlineServers=0 2024-11-21T11:14:42,486 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7b462513bfc2,38843,1732187670157' ***** 2024-11-21T11:14:42,486 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T11:14:42,487 INFO [M:0;7b462513bfc2:38843 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T11:14:42,487 INFO [M:0;7b462513bfc2:38843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T11:14:42,487 DEBUG [M:0;7b462513bfc2:38843 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T11:14:42,487 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T11:14:42,487 DEBUG [M:0;7b462513bfc2:38843 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T11:14:42,487 DEBUG [master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.large.0-1732187672774 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.large.0-1732187672774,5,FailOnTimeoutGroup] 2024-11-21T11:14:42,487 INFO [M:0;7b462513bfc2:38843 {}] hbase.ChoreService(370): Chore service for: master/7b462513bfc2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T11:14:42,487 INFO [M:0;7b462513bfc2:38843 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T11:14:42,487 DEBUG [M:0;7b462513bfc2:38843 {}] master.HMaster(1795): Stopping service threads 2024-11-21T11:14:42,488 INFO [M:0;7b462513bfc2:38843 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T11:14:42,488 DEBUG [master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.small.0-1732187672775 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b462513bfc2:0:becomeActiveMaster-HFileCleaner.small.0-1732187672775,5,FailOnTimeoutGroup] 2024-11-21T11:14:42,488 INFO [M:0;7b462513bfc2:38843 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T11:14:42,488 INFO [M:0;7b462513bfc2:38843 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T11:14:42,488 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T11:14:42,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T11:14:42,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T11:14:42,490 DEBUG [M:0;7b462513bfc2:38843 {}] zookeeper.ZKUtil(347): master:38843-0x1013a3c2d4c0000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T11:14:42,490 WARN [M:0;7b462513bfc2:38843 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T11:14:42,491 INFO [M:0;7b462513bfc2:38843 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/.lastflushedseqids 2024-11-21T11:14:42,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741870_1046 (size=1) 2024-11-21T11:14:42,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741870_1046 (size=1) 2024-11-21T11:14:42,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741870_1046 (size=1) 2024-11-21T11:14:42,502 INFO [M:0;7b462513bfc2:38843 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T11:14:42,502 INFO [M:0;7b462513bfc2:38843 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T11:14:42,502 DEBUG [M:0;7b462513bfc2:38843 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T11:14:42,503 INFO [M:0;7b462513bfc2:38843 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:42,503 DEBUG [M:0;7b462513bfc2:38843 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:42,503 DEBUG [M:0;7b462513bfc2:38843 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T11:14:42,503 DEBUG [M:0;7b462513bfc2:38843 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T11:14:42,503 INFO [M:0;7b462513bfc2:38843 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=260.85 KB heapSize=317.05 KB 2024-11-21T11:14:42,526 DEBUG [M:0;7b462513bfc2:38843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/625e026e4cd14812aca693db975139b6 is 82, key is hbase:meta,,1/info:regioninfo/1732187680980/Put/seqid=0 2024-11-21T11:14:42,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39687 is added to blk_1073741871_1047 (size=6071) 2024-11-21T11:14:42,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741871_1047 (size=6071) 2024-11-21T11:14:42,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33187 is added to blk_1073741871_1047 (size=6071) 2024-11-21T11:14:42,534 INFO [M:0;7b462513bfc2:38843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/625e026e4cd14812aca693db975139b6 2024-11-21T11:14:42,557 DEBUG [M:0;7b462513bfc2:38843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41741/user/jenkins/test-data/03bae55c-0f94-f944-c0e8-c408c22dd555/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/79648c450ccb40a1aeabb6f8bf38d874 is 2990, key is \x00\x00\x00\x00\x00\x00\x00