2024-11-07 11:13:30,362 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-07 11:13:30,374 main DEBUG Took 0.009885 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 11:13:30,375 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 11:13:30,375 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 11:13:30,376 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 11:13:30,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,384 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 11:13:30,396 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,397 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,398 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,399 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,399 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,400 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,400 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,401 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,401 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,402 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,402 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,403 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,404 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,404 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,404 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,405 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,406 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,407 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:13:30,408 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,408 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 11:13:30,410 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:13:30,412 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 11:13:30,414 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 11:13:30,415 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 11:13:30,417 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 11:13:30,417 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 11:13:30,428 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 11:13:30,432 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 11:13:30,434 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 11:13:30,435 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 11:13:30,435 main DEBUG createAppenders(={Console}) 2024-11-07 11:13:30,436 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-07 11:13:30,437 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-07 11:13:30,437 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-07 11:13:30,438 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 11:13:30,438 main DEBUG OutputStream closed 2024-11-07 11:13:30,439 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 11:13:30,439 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 11:13:30,439 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-07 11:13:30,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 11:13:30,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 11:13:30,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 11:13:30,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 11:13:30,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 11:13:30,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 11:13:30,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 11:13:30,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 11:13:30,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 11:13:30,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 11:13:30,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 11:13:30,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 11:13:30,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 11:13:30,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 11:13:30,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 11:13:30,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 11:13:30,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 11:13:30,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 11:13:30,525 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 11:13:30,526 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-07 11:13:30,526 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 11:13:30,527 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-07T11:13:30,539 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-07 11:13:30,542 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 11:13:30,542 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T11:13:30,759 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53 2024-11-07T11:13:30,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T11:13:30,792 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25, deleteOnExit=true 2024-11-07T11:13:30,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T11:13:30,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/test.cache.data in system properties and HBase conf 2024-11-07T11:13:30,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T11:13:30,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir in system properties and HBase conf 2024-11-07T11:13:30,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T11:13:30,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T11:13:30,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T11:13:30,884 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T11:13:30,976 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:13:30,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T11:13:30,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T11:13:30,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T11:13:30,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T11:13:30,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T11:13:30,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T11:13:30,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T11:13:30,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T11:13:30,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T11:13:30,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/nfs.dump.dir in system properties and HBase conf 2024-11-07T11:13:30,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/java.io.tmpdir in system properties and HBase conf 2024-11-07T11:13:30,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T11:13:30,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T11:13:30,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T11:13:32,023 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T11:13:32,091 INFO [Time-limited test {}] log.Log(170): Logging initialized @2441ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T11:13:32,158 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T11:13:32,213 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T11:13:32,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T11:13:32,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T11:13:32,235 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T11:13:32,247 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T11:13:32,250 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,AVAILABLE} 2024-11-07T11:13:32,251 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T11:13:32,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/java.io.tmpdir/jetty-localhost-35771-hadoop-hdfs-3_4_1-tests_jar-_-any-4771933267308495074/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T11:13:32,439 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:35771} 2024-11-07T11:13:32,439 INFO [Time-limited test {}] server.Server(415): Started @2790ms 2024-11-07T11:13:33,022 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T11:13:33,030 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T11:13:33,031 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T11:13:33,032 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T11:13:33,032 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T11:13:33,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,AVAILABLE} 2024-11-07T11:13:33,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T11:13:33,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/java.io.tmpdir/jetty-localhost-46653-hadoop-hdfs-3_4_1-tests_jar-_-any-7355868307880838037/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T11:13:33,151 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:46653} 2024-11-07T11:13:33,151 INFO [Time-limited test {}] server.Server(415): Started @3502ms 2024-11-07T11:13:33,203 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T11:13:33,323 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T11:13:33,330 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T11:13:33,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T11:13:33,333 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T11:13:33,333 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T11:13:33,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,AVAILABLE} 2024-11-07T11:13:33,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T11:13:33,478 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/java.io.tmpdir/jetty-localhost-45719-hadoop-hdfs-3_4_1-tests_jar-_-any-14911421487985703551/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T11:13:33,479 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:45719} 2024-11-07T11:13:33,479 INFO [Time-limited test {}] server.Server(415): Started @3830ms 2024-11-07T11:13:33,481 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T11:13:33,521 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T11:13:33,525 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T11:13:33,526 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T11:13:33,527 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T11:13:33,527 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T11:13:33,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,AVAILABLE} 2024-11-07T11:13:33,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T11:13:33,627 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/java.io.tmpdir/jetty-localhost-39779-hadoop-hdfs-3_4_1-tests_jar-_-any-6815691183736614431/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T11:13:33,628 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:39779} 2024-11-07T11:13:33,628 INFO [Time-limited test {}] server.Server(415): Started @3979ms 2024-11-07T11:13:33,630 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T11:13:34,938 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data1/current/BP-1322027227-172.17.0.2-1730978011492/current, will proceed with Du for space computation calculation, 2024-11-07T11:13:34,938 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data2/current/BP-1322027227-172.17.0.2-1730978011492/current, will proceed with Du for space computation calculation, 2024-11-07T11:13:34,938 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data4/current/BP-1322027227-172.17.0.2-1730978011492/current, will proceed with Du for space computation calculation, 2024-11-07T11:13:34,938 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data3/current/BP-1322027227-172.17.0.2-1730978011492/current, will proceed with Du for space computation calculation, 2024-11-07T11:13:34,972 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T11:13:34,972 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T11:13:35,019 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b93a5a1dc51a9a2 with lease ID 0x7b92360553cc0584: Processing first storage report for DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab from datanode DatanodeRegistration(127.0.0.1:44443, datanodeUuid=d269a19d-938c-44cc-84f8-95f4302f4f7c, infoPort=39503, infoSecurePort=0, ipcPort=34529, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492) 2024-11-07T11:13:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b93a5a1dc51a9a2 with lease ID 0x7b92360553cc0584: from storage DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab node DatanodeRegistration(127.0.0.1:44443, datanodeUuid=d269a19d-938c-44cc-84f8-95f4302f4f7c, infoPort=39503, infoSecurePort=0, ipcPort=34529, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T11:13:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedb991c8be2ce864 with lease ID 0x7b92360553cc0585: Processing first storage report for DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf from datanode DatanodeRegistration(127.0.0.1:35325, datanodeUuid=bc2c6651-5d3f-4923-ba9c-d15aeada7b4c, infoPort=45603, infoSecurePort=0, ipcPort=46183, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492) 2024-11-07T11:13:35,022 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedb991c8be2ce864 with lease ID 0x7b92360553cc0585: from storage DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf node DatanodeRegistration(127.0.0.1:35325, datanodeUuid=bc2c6651-5d3f-4923-ba9c-d15aeada7b4c, infoPort=45603, infoSecurePort=0, ipcPort=46183, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T11:13:35,022 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b93a5a1dc51a9a2 with lease ID 0x7b92360553cc0584: Processing first storage report for DS-21aa3022-169c-4e4c-8362-ff7722fd98f9 from datanode DatanodeRegistration(127.0.0.1:44443, datanodeUuid=d269a19d-938c-44cc-84f8-95f4302f4f7c, infoPort=39503, infoSecurePort=0, ipcPort=34529, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492) 2024-11-07T11:13:35,022 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b93a5a1dc51a9a2 with lease ID 0x7b92360553cc0584: from storage DS-21aa3022-169c-4e4c-8362-ff7722fd98f9 node DatanodeRegistration(127.0.0.1:44443, datanodeUuid=d269a19d-938c-44cc-84f8-95f4302f4f7c, infoPort=39503, infoSecurePort=0, ipcPort=34529, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T11:13:35,023 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedb991c8be2ce864 with lease ID 0x7b92360553cc0585: Processing first storage report for DS-42258e67-2e5c-4334-a2c6-c78d9c2e4f0e from datanode DatanodeRegistration(127.0.0.1:35325, datanodeUuid=bc2c6651-5d3f-4923-ba9c-d15aeada7b4c, infoPort=45603, infoSecurePort=0, ipcPort=46183, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492) 2024-11-07T11:13:35,023 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedb991c8be2ce864 with lease ID 0x7b92360553cc0585: from storage DS-42258e67-2e5c-4334-a2c6-c78d9c2e4f0e node DatanodeRegistration(127.0.0.1:35325, datanodeUuid=bc2c6651-5d3f-4923-ba9c-d15aeada7b4c, infoPort=45603, infoSecurePort=0, ipcPort=46183, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T11:13:35,156 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data6/current/BP-1322027227-172.17.0.2-1730978011492/current, will proceed with Du for space computation calculation, 2024-11-07T11:13:35,156 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data5/current/BP-1322027227-172.17.0.2-1730978011492/current, will proceed with Du for space computation calculation, 2024-11-07T11:13:35,175 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T11:13:35,180 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5ce2f904d0dee47 with lease ID 0x7b92360553cc0586: Processing first storage report for DS-46fe5871-e6c7-4d1d-b05c-860d66f04292 from datanode DatanodeRegistration(127.0.0.1:45515, datanodeUuid=3a9dbdde-f6ec-4c05-bd5c-a57bcc0cae55, infoPort=42855, infoSecurePort=0, ipcPort=45531, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492) 2024-11-07T11:13:35,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5ce2f904d0dee47 with lease ID 0x7b92360553cc0586: from storage DS-46fe5871-e6c7-4d1d-b05c-860d66f04292 node DatanodeRegistration(127.0.0.1:45515, datanodeUuid=3a9dbdde-f6ec-4c05-bd5c-a57bcc0cae55, infoPort=42855, infoSecurePort=0, ipcPort=45531, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T11:13:35,181 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5ce2f904d0dee47 with lease ID 0x7b92360553cc0586: Processing first storage report for DS-6e55d7c6-cd03-4f14-884a-cc75fd284f02 from datanode DatanodeRegistration(127.0.0.1:45515, datanodeUuid=3a9dbdde-f6ec-4c05-bd5c-a57bcc0cae55, infoPort=42855, infoSecurePort=0, ipcPort=45531, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492) 2024-11-07T11:13:35,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5ce2f904d0dee47 with lease ID 0x7b92360553cc0586: from storage DS-6e55d7c6-cd03-4f14-884a-cc75fd284f02 node DatanodeRegistration(127.0.0.1:45515, datanodeUuid=3a9dbdde-f6ec-4c05-bd5c-a57bcc0cae55, infoPort=42855, infoSecurePort=0, ipcPort=45531, storageInfo=lv=-57;cid=testClusterID;nsid=1626404671;c=1730978011492), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T11:13:35,267 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53 2024-11-07T11:13:35,331 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/zookeeper_0, clientPort=54468, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T11:13:35,340 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54468 2024-11-07T11:13:35,352 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:35,354 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:35,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741825_1001 (size=7) 2024-11-07T11:13:35,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741825_1001 (size=7) 2024-11-07T11:13:35,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741825_1001 (size=7) 2024-11-07T11:13:35,955 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab with version=8 2024-11-07T11:13:35,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/hbase-staging 2024-11-07T11:13:36,028 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T11:13:36,254 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4ab3d709c6df:0 server-side Connection retries=45 2024-11-07T11:13:36,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,267 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:13:36,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:13:36,384 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T11:13:36,432 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T11:13:36,440 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T11:13:36,443 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:13:36,465 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3746 (auto-detected) 2024-11-07T11:13:36,465 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T11:13:36,481 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37891 2024-11-07T11:13:36,500 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37891 connecting to ZooKeeper ensemble=127.0.0.1:54468 2024-11-07T11:13:36,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378910x0, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:13:36,625 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37891-0x101150063ff0000 connected 2024-11-07T11:13:36,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,720 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:36,733 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab, hbase.cluster.distributed=false 2024-11-07T11:13:36,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:13:36,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37891 2024-11-07T11:13:36,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37891 2024-11-07T11:13:36,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37891 2024-11-07T11:13:36,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37891 2024-11-07T11:13:36,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37891 2024-11-07T11:13:36,850 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4ab3d709c6df:0 server-side Connection retries=45 2024-11-07T11:13:36,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,852 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:13:36,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:13:36,854 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:13:36,858 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:13:36,859 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43027 2024-11-07T11:13:36,861 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43027 connecting to ZooKeeper ensemble=127.0.0.1:54468 2024-11-07T11:13:36,862 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,864 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430270x0, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:13:36,881 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430270x0, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:36,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43027-0x101150063ff0001 connected 2024-11-07T11:13:36,885 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:13:36,891 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:13:36,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:13:36,898 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:13:36,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43027 2024-11-07T11:13:36,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43027 2024-11-07T11:13:36,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43027 2024-11-07T11:13:36,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43027 2024-11-07T11:13:36,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43027 2024-11-07T11:13:36,917 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4ab3d709c6df:0 server-side Connection retries=45 2024-11-07T11:13:36,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,918 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:13:36,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:13:36,919 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:13:36,919 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:13:36,920 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43401 2024-11-07T11:13:36,921 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43401 connecting to ZooKeeper ensemble=127.0.0.1:54468 2024-11-07T11:13:36,922 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434010x0, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:13:36,937 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434010x0, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:36,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43401-0x101150063ff0002 connected 2024-11-07T11:13:36,938 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:13:36,940 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:13:36,941 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:13:36,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:13:36,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43401 2024-11-07T11:13:36,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43401 2024-11-07T11:13:36,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43401 2024-11-07T11:13:36,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43401 2024-11-07T11:13:36,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43401 2024-11-07T11:13:36,967 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4ab3d709c6df:0 server-side Connection retries=45 2024-11-07T11:13:36,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,968 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:13:36,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:13:36,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:13:36,969 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:13:36,969 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:13:36,970 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33951 2024-11-07T11:13:36,971 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33951 connecting to ZooKeeper ensemble=127.0.0.1:54468 2024-11-07T11:13:36,973 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,975 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:36,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339510x0, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:13:36,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:339510x0, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:36,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33951-0x101150063ff0003 connected 2024-11-07T11:13:36,990 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:13:36,991 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:13:36,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:13:36,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:13:36,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33951 2024-11-07T11:13:36,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33951 2024-11-07T11:13:36,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33951 2024-11-07T11:13:37,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33951 2024-11-07T11:13:37,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33951 2024-11-07T11:13:37,021 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4ab3d709c6df:37891 2024-11-07T11:13:37,022 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,041 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:13:37,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:13:37,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:13:37,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,071 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T11:13:37,073 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4ab3d709c6df,37891,1730978016105 from backup master directory 2024-11-07T11:13:37,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:13:37,085 WARN [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:13:37,085 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,089 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T11:13:37,091 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T11:13:37,145 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/hbase.id] with ID: f2770213-a4f6-4295-bb6b-8f027a7a79d8 2024-11-07T11:13:37,145 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/.tmp/hbase.id 2024-11-07T11:13:37,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741826_1002 (size=42) 2024-11-07T11:13:37,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741826_1002 (size=42) 2024-11-07T11:13:37,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741826_1002 (size=42) 2024-11-07T11:13:37,162 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/.tmp/hbase.id]:[hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/hbase.id] 2024-11-07T11:13:37,206 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T11:13:37,211 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T11:13:37,229 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-07T11:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741827_1003 (size=196) 2024-11-07T11:13:37,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741827_1003 (size=196) 2024-11-07T11:13:37,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741827_1003 (size=196) 2024-11-07T11:13:37,278 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T11:13:37,279 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T11:13:37,292 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T11:13:37,296 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:13:37,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741828_1004 (size=1189) 2024-11-07T11:13:37,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741828_1004 (size=1189) 2024-11-07T11:13:37,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741828_1004 (size=1189) 2024-11-07T11:13:37,343 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store 2024-11-07T11:13:37,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741829_1005 (size=34) 2024-11-07T11:13:37,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741829_1005 (size=34) 2024-11-07T11:13:37,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741829_1005 (size=34) 2024-11-07T11:13:37,368 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T11:13:37,371 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:37,373 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T11:13:37,373 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:37,373 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:37,375 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T11:13:37,375 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:37,375 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:37,376 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730978017372Disabling compacts and flushes for region at 1730978017372Disabling writes for close at 1730978017375 (+3 ms)Writing region close event to WAL at 1730978017375Closed at 1730978017375 2024-11-07T11:13:37,379 WARN [master/4ab3d709c6df:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/.initializing 2024-11-07T11:13:37,379 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/WALs/4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,386 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:13:37,402 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4ab3d709c6df%2C37891%2C1730978016105, suffix=, logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/WALs/4ab3d709c6df,37891,1730978016105, archiveDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/oldWALs, maxLogs=10 2024-11-07T11:13:37,432 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/WALs/4ab3d709c6df,37891,1730978016105/4ab3d709c6df%2C37891%2C1730978016105.1730978017408, exclude list is [], retry=0 2024-11-07T11:13:37,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44443,DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab,DISK] 2024-11-07T11:13:37,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45515,DS-46fe5871-e6c7-4d1d-b05c-860d66f04292,DISK] 2024-11-07T11:13:37,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35325,DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf,DISK] 2024-11-07T11:13:37,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T11:13:37,485 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/WALs/4ab3d709c6df,37891,1730978016105/4ab3d709c6df%2C37891%2C1730978016105.1730978017408 2024-11-07T11:13:37,485 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39503:39503),(127.0.0.1/127.0.0.1:45603:45603),(127.0.0.1/127.0.0.1:42855:42855)] 2024-11-07T11:13:37,486 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:13:37,486 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:37,489 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,490 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T11:13:37,548 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:37,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:37,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T11:13:37,555 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:37,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:37,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,559 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T11:13:37,559 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:37,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:37,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,563 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T11:13:37,563 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:37,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:37,564 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,568 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,569 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,573 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,574 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,578 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:13:37,581 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:13:37,585 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:37,586 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75492478, jitterRate=0.12492558360099792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:13:37,597 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730978017500Initializing all the Stores at 1730978017502 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978017503 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978017503Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978017504 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978017504Cleaning up temporary data from old regions at 1730978017574 (+70 ms)Region opened successfully at 1730978017597 (+23 ms) 2024-11-07T11:13:37,598 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T11:13:37,631 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297c15ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4ab3d709c6df/172.17.0.2:0 2024-11-07T11:13:37,658 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T11:13:37,667 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:37,668 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:37,670 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:37,671 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T11:13:37,676 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T11:13:37,676 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T11:13:37,700 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T11:13:37,707 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T11:13:37,757 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T11:13:37,760 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T11:13:37,761 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T11:13:37,775 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T11:13:37,778 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T11:13:37,783 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T11:13:37,795 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T11:13:37,797 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T11:13:37,810 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T11:13:37,829 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T11:13:37,841 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T11:13:37,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:37,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:37,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:37,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:37,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,858 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4ab3d709c6df,37891,1730978016105, sessionid=0x101150063ff0000, setting cluster-up flag (Was=false) 2024-11-07T11:13:37,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,926 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T11:13:37,928 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:37,989 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T11:13:37,992 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:37,999 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T11:13:38,007 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(746): ClusterId : f2770213-a4f6-4295-bb6b-8f027a7a79d8 2024-11-07T11:13:38,010 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(746): ClusterId : f2770213-a4f6-4295-bb6b-8f027a7a79d8 2024-11-07T11:13:38,011 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:13:38,011 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:13:38,011 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(746): ClusterId : f2770213-a4f6-4295-bb6b-8f027a7a79d8 2024-11-07T11:13:38,011 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:13:38,034 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:13:38,034 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:13:38,035 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:13:38,035 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:13:38,048 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:13:38,048 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:13:38,059 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:13:38,059 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:13:38,060 DEBUG [RS:1;4ab3d709c6df:43401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e7430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4ab3d709c6df/172.17.0.2:0 2024-11-07T11:13:38,060 DEBUG [RS:0;4ab3d709c6df:43027 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45be6fba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4ab3d709c6df/172.17.0.2:0 2024-11-07T11:13:38,060 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:13:38,060 DEBUG [RS:2;4ab3d709c6df:33951 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af31788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4ab3d709c6df/172.17.0.2:0 2024-11-07T11:13:38,078 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4ab3d709c6df:43027 2024-11-07T11:13:38,082 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4ab3d709c6df:43401 2024-11-07T11:13:38,082 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4ab3d709c6df:33951 2024-11-07T11:13:38,088 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:13:38,089 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:13:38,089 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:13:38,089 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:13:38,089 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:13:38,089 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:13:38,089 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:13:38,089 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:13:38,089 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:13:38,092 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(2659): reportForDuty to master=4ab3d709c6df,37891,1730978016105 with port=43027, startcode=1730978016820 2024-11-07T11:13:38,092 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(2659): reportForDuty to master=4ab3d709c6df,37891,1730978016105 with port=43401, startcode=1730978016916 2024-11-07T11:13:38,092 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(2659): reportForDuty to master=4ab3d709c6df,37891,1730978016105 with port=33951, startcode=1730978016967 2024-11-07T11:13:38,096 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:38,103 DEBUG [RS:0;4ab3d709c6df:43027 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:13:38,103 DEBUG [RS:1;4ab3d709c6df:43401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:13:38,103 DEBUG [RS:2;4ab3d709c6df:33951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:13:38,105 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T11:13:38,112 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T11:13:38,118 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4ab3d709c6df,37891,1730978016105 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T11:13:38,128 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4ab3d709c6df:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:13:38,128 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4ab3d709c6df:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:13:38,128 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4ab3d709c6df:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:13:38,128 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4ab3d709c6df:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:13:38,129 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4ab3d709c6df:0, corePoolSize=10, maxPoolSize=10 2024-11-07T11:13:38,129 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,129 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4ab3d709c6df:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:13:38,129 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,130 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978048130 2024-11-07T11:13:38,132 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T11:13:38,133 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T11:13:38,136 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:38,137 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T11:13:38,138 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:13:38,138 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46881, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:13:38,138 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37865, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:13:38,139 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T11:13:38,140 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T11:13:38,140 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T11:13:38,140 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T11:13:38,143 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:38,141 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,143 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T11:13:38,146 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:38,147 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T11:13:38,149 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T11:13:38,149 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T11:13:38,152 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T11:13:38,153 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T11:13:38,153 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:38,154 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:38,154 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4ab3d709c6df:0:becomeActiveMaster-HFileCleaner.large.0-1730978018154,5,FailOnTimeoutGroup] 2024-11-07T11:13:38,156 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4ab3d709c6df:0:becomeActiveMaster-HFileCleaner.small.0-1730978018155,5,FailOnTimeoutGroup] 2024-11-07T11:13:38,156 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,156 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T11:13:38,158 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,158 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741831_1007 (size=1321) 2024-11-07T11:13:38,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741831_1007 (size=1321) 2024-11-07T11:13:38,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741831_1007 (size=1321) 2024-11-07T11:13:38,201 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T11:13:38,201 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:38,210 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T11:13:38,210 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T11:13:38,210 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T11:13:38,210 WARN [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T11:13:38,210 WARN [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T11:13:38,210 WARN [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T11:13:38,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741832_1008 (size=32) 2024-11-07T11:13:38,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741832_1008 (size=32) 2024-11-07T11:13:38,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741832_1008 (size=32) 2024-11-07T11:13:38,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:38,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T11:13:38,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:13:38,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:38,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:38,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T11:13:38,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:13:38,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:38,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:38,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T11:13:38,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:13:38,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:38,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:38,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T11:13:38,232 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:13:38,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:38,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:38,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:13:38,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740 2024-11-07T11:13:38,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740 2024-11-07T11:13:38,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:13:38,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:13:38,240 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:13:38,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:13:38,246 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:38,247 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65287435, jitterRate=-0.027141407132148743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:13:38,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730978018217Initializing all the Stores at 1730978018218 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978018218Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978018219 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978018219Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978018219Cleaning up temporary data from old regions at 1730978018239 (+20 ms)Region opened successfully at 1730978018250 (+11 ms) 2024-11-07T11:13:38,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T11:13:38,251 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T11:13:38,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T11:13:38,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T11:13:38,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T11:13:38,252 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T11:13:38,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730978018251Disabling compacts and flushes for region at 1730978018251Disabling writes for close at 1730978018251Writing region close event to WAL at 1730978018252 (+1 ms)Closed at 1730978018252 2024-11-07T11:13:38,255 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:38,256 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T11:13:38,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T11:13:38,270 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:38,273 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T11:13:38,311 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(2659): reportForDuty to master=4ab3d709c6df,37891,1730978016105 with port=33951, startcode=1730978016967 2024-11-07T11:13:38,311 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(2659): reportForDuty to master=4ab3d709c6df,37891,1730978016105 with port=43401, startcode=1730978016916 2024-11-07T11:13:38,311 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(2659): reportForDuty to master=4ab3d709c6df,37891,1730978016105 with port=43027, startcode=1730978016820 2024-11-07T11:13:38,314 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] master.ServerManager(517): Registering regionserver=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,325 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:38,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] master.ServerManager(517): Registering regionserver=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:38,326 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:38,326 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37645 2024-11-07T11:13:38,326 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:13:38,328 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:38,328 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:38,328 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37645 2024-11-07T11:13:38,328 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] master.ServerManager(517): Registering regionserver=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:38,328 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:13:38,331 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:38,331 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37645 2024-11-07T11:13:38,331 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:13:38,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:13:38,378 DEBUG [RS:0;4ab3d709c6df:43027 {}] zookeeper.ZKUtil(111): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,379 WARN [RS:0;4ab3d709c6df:43027 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:13:38,379 DEBUG [RS:2;4ab3d709c6df:33951 {}] zookeeper.ZKUtil(111): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:38,379 DEBUG [RS:1;4ab3d709c6df:43401 {}] zookeeper.ZKUtil(111): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:38,379 INFO [RS:0;4ab3d709c6df:43027 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:13:38,379 WARN [RS:2;4ab3d709c6df:33951 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:13:38,379 WARN [RS:1;4ab3d709c6df:43401 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:13:38,379 INFO [RS:2;4ab3d709c6df:33951 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:13:38,379 INFO [RS:1;4ab3d709c6df:43401 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:13:38,379 DEBUG [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,379 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:38,379 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:38,381 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4ab3d709c6df,43401,1730978016916] 2024-11-07T11:13:38,381 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4ab3d709c6df,33951,1730978016967] 2024-11-07T11:13:38,381 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4ab3d709c6df,43027,1730978016820] 2024-11-07T11:13:38,409 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:13:38,409 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:13:38,409 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:13:38,424 WARN [4ab3d709c6df:37891 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-07T11:13:38,425 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:13:38,425 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:13:38,425 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:13:38,430 INFO [RS:1;4ab3d709c6df:43401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:13:38,430 INFO [RS:0;4ab3d709c6df:43027 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:13:38,430 INFO [RS:2;4ab3d709c6df:33951 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:13:38,431 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,431 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,431 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,431 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:13:38,431 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:13:38,431 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:13:38,437 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:13:38,437 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:13:38,437 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:13:38,438 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,438 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,438 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,439 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,439 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4ab3d709c6df:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:13:38,440 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4ab3d709c6df:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:13:38,441 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4ab3d709c6df:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:13:38,441 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4ab3d709c6df:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:13:38,441 DEBUG [RS:2;4ab3d709c6df:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4ab3d709c6df:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:13:38,441 DEBUG [RS:1;4ab3d709c6df:43401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4ab3d709c6df:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:13:38,441 DEBUG [RS:0;4ab3d709c6df:43027 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4ab3d709c6df:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:13:38,442 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,442 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,33951,1730978016967-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,43401,1730978016916-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:13:38,443 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,43027,1730978016820-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:13:38,469 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:13:38,469 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:13:38,469 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:13:38,471 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,43027,1730978016820-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,471 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,43401,1730978016916-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,471 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,33951,1730978016967-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,472 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,472 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,472 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,472 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.Replication(171): 4ab3d709c6df,33951,1730978016967 started 2024-11-07T11:13:38,472 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.Replication(171): 4ab3d709c6df,43027,1730978016820 started 2024-11-07T11:13:38,472 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.Replication(171): 4ab3d709c6df,43401,1730978016916 started 2024-11-07T11:13:38,496 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,496 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,496 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:38,497 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1482): Serving as 4ab3d709c6df,33951,1730978016967, RpcServer on 4ab3d709c6df/172.17.0.2:33951, sessionid=0x101150063ff0003 2024-11-07T11:13:38,497 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(1482): Serving as 4ab3d709c6df,43027,1730978016820, RpcServer on 4ab3d709c6df/172.17.0.2:43027, sessionid=0x101150063ff0001 2024-11-07T11:13:38,497 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1482): Serving as 4ab3d709c6df,43401,1730978016916, RpcServer on 4ab3d709c6df/172.17.0.2:43401, sessionid=0x101150063ff0002 2024-11-07T11:13:38,498 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:13:38,498 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:13:38,498 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:13:38,498 DEBUG [RS:2;4ab3d709c6df:33951 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:38,498 DEBUG [RS:1;4ab3d709c6df:43401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:38,498 DEBUG [RS:0;4ab3d709c6df:43027 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,498 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4ab3d709c6df,43027,1730978016820' 2024-11-07T11:13:38,498 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4ab3d709c6df,33951,1730978016967' 2024-11-07T11:13:38,498 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4ab3d709c6df,43401,1730978016916' 2024-11-07T11:13:38,498 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:13:38,498 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:13:38,498 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:13:38,499 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:13:38,499 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:13:38,499 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:13:38,500 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:13:38,500 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:13:38,500 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:13:38,500 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:13:38,500 DEBUG [RS:1;4ab3d709c6df:43401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:38,500 DEBUG [RS:0;4ab3d709c6df:43027 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,500 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4ab3d709c6df,43027,1730978016820' 2024-11-07T11:13:38,500 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4ab3d709c6df,43401,1730978016916' 2024-11-07T11:13:38,500 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:13:38,500 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:13:38,501 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:13:38,501 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:13:38,501 DEBUG [RS:2;4ab3d709c6df:33951 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:38,501 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4ab3d709c6df,33951,1730978016967' 2024-11-07T11:13:38,501 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:13:38,501 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:13:38,501 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:13:38,501 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:13:38,502 DEBUG [RS:1;4ab3d709c6df:43401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:13:38,502 DEBUG [RS:0;4ab3d709c6df:43027 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:13:38,502 INFO [RS:1;4ab3d709c6df:43401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:13:38,502 INFO [RS:0;4ab3d709c6df:43027 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:13:38,502 INFO [RS:1;4ab3d709c6df:43401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:13:38,502 INFO [RS:0;4ab3d709c6df:43027 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:13:38,502 DEBUG [RS:2;4ab3d709c6df:33951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:13:38,502 INFO [RS:2;4ab3d709c6df:33951 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:13:38,502 INFO [RS:2;4ab3d709c6df:33951 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:13:38,608 INFO [RS:2;4ab3d709c6df:33951 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:13:38,608 INFO [RS:0;4ab3d709c6df:43027 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:13:38,608 INFO [RS:1;4ab3d709c6df:43401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:13:38,612 INFO [RS:2;4ab3d709c6df:33951 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4ab3d709c6df%2C33951%2C1730978016967, suffix=, logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,33951,1730978016967, archiveDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs, maxLogs=32 2024-11-07T11:13:38,612 INFO [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4ab3d709c6df%2C43401%2C1730978016916, suffix=, logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916, archiveDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs, maxLogs=32 2024-11-07T11:13:38,612 INFO [RS:0;4ab3d709c6df:43027 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4ab3d709c6df%2C43027%2C1730978016820, suffix=, logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820, archiveDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs, maxLogs=32 2024-11-07T11:13:38,630 DEBUG [RS:0;4ab3d709c6df:43027 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820/4ab3d709c6df%2C43027%2C1730978016820.1730978018617, exclude list is [], retry=0 2024-11-07T11:13:38,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35325,DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf,DISK] 2024-11-07T11:13:38,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45515,DS-46fe5871-e6c7-4d1d-b05c-860d66f04292,DISK] 2024-11-07T11:13:38,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44443,DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab,DISK] 2024-11-07T11:13:38,637 DEBUG [RS:1;4ab3d709c6df:43401 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916/4ab3d709c6df%2C43401%2C1730978016916.1730978018617, exclude list is [], retry=0 2024-11-07T11:13:38,637 DEBUG [RS:2;4ab3d709c6df:33951 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,33951,1730978016967/4ab3d709c6df%2C33951%2C1730978016967.1730978018617, exclude list is [], retry=0 2024-11-07T11:13:38,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45515,DS-46fe5871-e6c7-4d1d-b05c-860d66f04292,DISK] 2024-11-07T11:13:38,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35325,DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf,DISK] 2024-11-07T11:13:38,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35325,DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf,DISK] 2024-11-07T11:13:38,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44443,DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab,DISK] 2024-11-07T11:13:38,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45515,DS-46fe5871-e6c7-4d1d-b05c-860d66f04292,DISK] 2024-11-07T11:13:38,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44443,DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab,DISK] 2024-11-07T11:13:38,650 INFO [RS:0;4ab3d709c6df:43027 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 2024-11-07T11:13:38,652 DEBUG [RS:0;4ab3d709c6df:43027 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39503:39503),(127.0.0.1/127.0.0.1:42855:42855),(127.0.0.1/127.0.0.1:45603:45603)] 2024-11-07T11:13:38,662 INFO [RS:2;4ab3d709c6df:33951 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,33951,1730978016967/4ab3d709c6df%2C33951%2C1730978016967.1730978018617 2024-11-07T11:13:38,665 DEBUG [RS:2;4ab3d709c6df:33951 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42855:42855),(127.0.0.1/127.0.0.1:45603:45603),(127.0.0.1/127.0.0.1:39503:39503)] 2024-11-07T11:13:38,667 INFO [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916/4ab3d709c6df%2C43401%2C1730978016916.1730978018617 2024-11-07T11:13:38,668 DEBUG [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45603:45603),(127.0.0.1/127.0.0.1:42855:42855),(127.0.0.1/127.0.0.1:39503:39503)] 2024-11-07T11:13:38,676 DEBUG [4ab3d709c6df:37891 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T11:13:38,688 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:38,694 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43027,1730978016820, state=OPENING 2024-11-07T11:13:38,789 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T11:13:38,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:38,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:38,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:38,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:38,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:38,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:38,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:38,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:38,805 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:38,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:38,988 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:13:38,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53561, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:13:39,002 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T11:13:39,002 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:13:39,003 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T11:13:39,007 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4ab3d709c6df%2C43027%2C1730978016820.meta, suffix=.meta, logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820, archiveDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs, maxLogs=32 2024-11-07T11:13:39,023 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, exclude list is [], retry=0 2024-11-07T11:13:39,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44443,DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab,DISK] 2024-11-07T11:13:39,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45515,DS-46fe5871-e6c7-4d1d-b05c-860d66f04292,DISK] 2024-11-07T11:13:39,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35325,DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf,DISK] 2024-11-07T11:13:39,032 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta 2024-11-07T11:13:39,035 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39503:39503),(127.0.0.1/127.0.0.1:42855:42855),(127.0.0.1/127.0.0.1:45603:45603)] 2024-11-07T11:13:39,035 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:13:39,037 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T11:13:39,040 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T11:13:39,044 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T11:13:39,048 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T11:13:39,049 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,049 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T11:13:39,049 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T11:13:39,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T11:13:39,055 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:13:39,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:39,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:39,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T11:13:39,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:13:39,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:39,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:39,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T11:13:39,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:13:39,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:39,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:39,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T11:13:39,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:13:39,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:39,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:39,067 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:13:39,069 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740 2024-11-07T11:13:39,072 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740 2024-11-07T11:13:39,075 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:13:39,075 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:13:39,076 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:13:39,079 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:13:39,081 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60819795, jitterRate=-0.09371443092823029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:13:39,081 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T11:13:39,082 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730978019050Writing region info on filesystem at 1730978019050Initializing all the Stores at 1730978019052 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978019052Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978019053 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978019053Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978019053Cleaning up temporary data from old regions at 1730978019075 (+22 ms)Running coprocessor post-open hooks at 1730978019081 (+6 ms)Region opened successfully at 1730978019082 (+1 ms) 2024-11-07T11:13:39,089 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730978018980 2024-11-07T11:13:39,102 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T11:13:39,102 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T11:13:39,104 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:39,107 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43027,1730978016820, state=OPEN 2024-11-07T11:13:39,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:39,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:39,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:39,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:39,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:39,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:39,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:39,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:39,197 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:39,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T11:13:39,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43027,1730978016820 in 389 msec 2024-11-07T11:13:39,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T11:13:39,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 945 msec 2024-11-07T11:13:39,214 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:39,214 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T11:13:39,233 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:13:39,234 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1] 2024-11-07T11:13:39,253 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:13:39,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39257, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:13:39,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2350 sec 2024-11-07T11:13:39,276 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730978019276, completionTime=-1 2024-11-07T11:13:39,279 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-07T11:13:39,279 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:39,302 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-07T11:13:39,303 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978079303 2024-11-07T11:13:39,303 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978139303 2024-11-07T11:13:39,303 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-07T11:13:39,305 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,37891,1730978016105-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,306 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,37891,1730978016105-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,306 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,37891,1730978016105-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,307 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4ab3d709c6df:37891, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,308 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,308 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,315 DEBUG [master/4ab3d709c6df:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T11:13:39,336 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.249sec 2024-11-07T11:13:39,337 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T11:13:39,338 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T11:13:39,339 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T11:13:39,340 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T11:13:39,340 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T11:13:39,340 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,37891,1730978016105-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:13:39,341 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,37891,1730978016105-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T11:13:39,349 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T11:13:39,349 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T11:13:39,350 INFO [master/4ab3d709c6df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4ab3d709c6df,37891,1730978016105-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:13:39,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:13:39,424 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T11:13:39,424 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T11:13:39,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4ab3d709c6df,37891,-1 for getting cluster id 2024-11-07T11:13:39,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T11:13:39,439 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f2770213-a4f6-4295-bb6b-8f027a7a79d8' 2024-11-07T11:13:39,442 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T11:13:39,442 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f2770213-a4f6-4295-bb6b-8f027a7a79d8" 2024-11-07T11:13:39,442 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb11c8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:13:39,442 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4ab3d709c6df,37891,-1] 2024-11-07T11:13:39,446 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T11:13:39,448 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:39,449 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T11:13:39,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:13:39,452 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:13:39,458 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1] 2024-11-07T11:13:39,458 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:13:39,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:13:39,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:39,483 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T11:13:39,487 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:39,490 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35863b9b 2024-11-07T11:13:39,490 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T11:13:39,493 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T11:13:39,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T11:13:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:39,535 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T11:13:39,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-07T11:13:39,538 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:39,541 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T11:13:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:13:39,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741837_1013 (size=358) 2024-11-07T11:13:39,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741837_1013 (size=358) 2024-11-07T11:13:39,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741837_1013 (size=358) 2024-11-07T11:13:39,571 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ebf8c75d5f559642dd7af8561a14d668, NAME => 'test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,572 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 908579c522ec42c02649f6f9625ec8a4, NAME => 'test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,572 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 880afe4596d2f9f36732bf50351d0096, NAME => 'test,,1730978019494.880afe4596d2f9f36732bf50351d0096.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741839_1015 (size=45) 2024-11-07T11:13:39,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741839_1015 (size=45) 2024-11-07T11:13:39,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741839_1015 (size=45) 2024-11-07T11:13:39,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 908579c522ec42c02649f6f9625ec8a4, disabling compactions & flushes 2024-11-07T11:13:39,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:39,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:39,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. after waiting 0 ms 2024-11-07T11:13:39,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:39,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:39,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 908579c522ec42c02649f6f9625ec8a4: Waiting for close lock at 1730978019597Disabling compacts and flushes for region at 1730978019597Disabling writes for close at 1730978019597Writing region close event to WAL at 1730978019597Closed at 1730978019597 2024-11-07T11:13:39,598 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e6670dae2fae722a1a46cdb36b5a703a, NAME => 'test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741838_1014 (size=45) 2024-11-07T11:13:39,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741838_1014 (size=45) 2024-11-07T11:13:39,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741838_1014 (size=45) 2024-11-07T11:13:39,605 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,605 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ebf8c75d5f559642dd7af8561a14d668, disabling compactions & flushes 2024-11-07T11:13:39,606 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:39,606 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:39,606 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. after waiting 0 ms 2024-11-07T11:13:39,606 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:39,606 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:39,606 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ebf8c75d5f559642dd7af8561a14d668: Waiting for close lock at 1730978019605Disabling compacts and flushes for region at 1730978019605Disabling writes for close at 1730978019606 (+1 ms)Writing region close event to WAL at 1730978019606Closed at 1730978019606 2024-11-07T11:13:39,607 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d7939d9f92fe5210293a2217ff7af696, NAME => 'test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741840_1016 (size=42) 2024-11-07T11:13:39,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741840_1016 (size=42) 2024-11-07T11:13:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741840_1016 (size=42) 2024-11-07T11:13:39,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1730978019494.880afe4596d2f9f36732bf50351d0096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 880afe4596d2f9f36732bf50351d0096, disabling compactions & flushes 2024-11-07T11:13:39,622 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:39,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:39,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1730978019494.880afe4596d2f9f36732bf50351d0096. after waiting 0 ms 2024-11-07T11:13:39,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:39,622 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:39,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 880afe4596d2f9f36732bf50351d0096: Waiting for close lock at 1730978019622Disabling compacts and flushes for region at 1730978019622Disabling writes for close at 1730978019622Writing region close event to WAL at 1730978019622Closed at 1730978019622 2024-11-07T11:13:39,623 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5b587e52af29a25529015bea97baa17, NAME => 'test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741841_1017 (size=45) 2024-11-07T11:13:39,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741841_1017 (size=45) 2024-11-07T11:13:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741841_1017 (size=45) 2024-11-07T11:13:39,632 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,632 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e6670dae2fae722a1a46cdb36b5a703a, disabling compactions & flushes 2024-11-07T11:13:39,632 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:39,632 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:39,632 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. after waiting 0 ms 2024-11-07T11:13:39,632 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:39,632 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:39,632 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e6670dae2fae722a1a46cdb36b5a703a: Waiting for close lock at 1730978019632Disabling compacts and flushes for region at 1730978019632Disabling writes for close at 1730978019632Writing region close event to WAL at 1730978019632Closed at 1730978019632 2024-11-07T11:13:39,633 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 356e319324497934b3703225e899ce57, NAME => 'test,ggg,1730978019494.356e319324497934b3703225e899ce57.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741842_1018 (size=45) 2024-11-07T11:13:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741843_1019 (size=45) 2024-11-07T11:13:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741843_1019 (size=45) 2024-11-07T11:13:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741843_1019 (size=45) 2024-11-07T11:13:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741842_1018 (size=45) 2024-11-07T11:13:39,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741842_1018 (size=45) 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d7939d9f92fe5210293a2217ff7af696, disabling compactions & flushes 2024-11-07T11:13:39,643 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. after waiting 0 ms 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:39,643 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d7939d9f92fe5210293a2217ff7af696: Waiting for close lock at 1730978019643Disabling compacts and flushes for region at 1730978019643Disabling writes for close at 1730978019643Writing region close event to WAL at 1730978019643Closed at 1730978019643 2024-11-07T11:13:39,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a5b587e52af29a25529015bea97baa17, disabling compactions & flushes 2024-11-07T11:13:39,643 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:39,644 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ba16fdeba546ef71df7c8b7d75e0fc39, NAME => 'test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:39,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. after waiting 0 ms 2024-11-07T11:13:39,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:39,644 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:39,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5b587e52af29a25529015bea97baa17: Waiting for close lock at 1730978019643Disabling compacts and flushes for region at 1730978019643Disabling writes for close at 1730978019644 (+1 ms)Writing region close event to WAL at 1730978019644Closed at 1730978019644 2024-11-07T11:13:39,645 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8c16ac0e25710f923881fe49bf532cf, NAME => 'test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741844_1020 (size=45) 2024-11-07T11:13:39,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741844_1020 (size=45) 2024-11-07T11:13:39,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741844_1020 (size=45) 2024-11-07T11:13:39,653 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1730978019494.356e319324497934b3703225e899ce57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,653 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 356e319324497934b3703225e899ce57, disabling compactions & flushes 2024-11-07T11:13:39,653 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:39,653 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:39,653 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1730978019494.356e319324497934b3703225e899ce57. after waiting 0 ms 2024-11-07T11:13:39,653 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:39,653 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:39,653 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 356e319324497934b3703225e899ce57: Waiting for close lock at 1730978019653Disabling compacts and flushes for region at 1730978019653Disabling writes for close at 1730978019653Writing region close event to WAL at 1730978019653Closed at 1730978019653 2024-11-07T11:13:39,654 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6356f249da7a2a5ae2d1836b77641b9d, NAME => 'test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:13:39,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741846_1022 (size=45) 2024-11-07T11:13:39,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741846_1022 (size=45) 2024-11-07T11:13:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741845_1021 (size=45) 2024-11-07T11:13:39,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741845_1021 (size=45) 2024-11-07T11:13:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741846_1022 (size=45) 2024-11-07T11:13:39,662 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,662 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f8c16ac0e25710f923881fe49bf532cf, disabling compactions & flushes 2024-11-07T11:13:39,662 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:39,662 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:39,662 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. after waiting 0 ms 2024-11-07T11:13:39,662 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:39,662 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:39,663 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8c16ac0e25710f923881fe49bf532cf: Waiting for close lock at 1730978019662Disabling compacts and flushes for region at 1730978019662Disabling writes for close at 1730978019662Writing region close event to WAL at 1730978019662Closed at 1730978019662 2024-11-07T11:13:39,664 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78b9a3cb232752746916dce9a515cf81, NAME => 'test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741845_1021 (size=45) 2024-11-07T11:13:39,665 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,666 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ba16fdeba546ef71df7c8b7d75e0fc39, disabling compactions & flushes 2024-11-07T11:13:39,666 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:39,666 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:39,666 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. after waiting 0 ms 2024-11-07T11:13:39,666 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:39,666 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:39,666 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ba16fdeba546ef71df7c8b7d75e0fc39: Waiting for close lock at 1730978019665Disabling compacts and flushes for region at 1730978019665Disabling writes for close at 1730978019666 (+1 ms)Writing region close event to WAL at 1730978019666Closed at 1730978019666 2024-11-07T11:13:39,667 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 314baf86ed26932b48f8c4cef53c1bcc, NAME => 'test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741847_1023 (size=45) 2024-11-07T11:13:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741847_1023 (size=45) 2024-11-07T11:13:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741847_1023 (size=45) 2024-11-07T11:13:39,675 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,675 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6356f249da7a2a5ae2d1836b77641b9d, disabling compactions & flushes 2024-11-07T11:13:39,675 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:39,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:39,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. after waiting 0 ms 2024-11-07T11:13:39,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:39,676 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:39,676 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6356f249da7a2a5ae2d1836b77641b9d: Waiting for close lock at 1730978019675Disabling compacts and flushes for region at 1730978019675Disabling writes for close at 1730978019676 (+1 ms)Writing region close event to WAL at 1730978019676Closed at 1730978019676 2024-11-07T11:13:39,677 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7aa9c0d5fa702e40d1908a62541f05e4, NAME => 'test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741848_1024 (size=45) 2024-11-07T11:13:39,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741848_1024 (size=45) 2024-11-07T11:13:39,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741848_1024 (size=45) 2024-11-07T11:13:39,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 78b9a3cb232752746916dce9a515cf81, disabling compactions & flushes 2024-11-07T11:13:39,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:39,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:39,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. after waiting 0 ms 2024-11-07T11:13:39,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:39,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:39,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78b9a3cb232752746916dce9a515cf81: Waiting for close lock at 1730978019685Disabling compacts and flushes for region at 1730978019685Disabling writes for close at 1730978019686 (+1 ms)Writing region close event to WAL at 1730978019686Closed at 1730978019686 2024-11-07T11:13:39,687 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1bffef86bb7015d7d1d3993e690e3a3c, NAME => 'test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741849_1025 (size=45) 2024-11-07T11:13:39,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741849_1025 (size=45) 2024-11-07T11:13:39,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741849_1025 (size=45) 2024-11-07T11:13:39,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741850_1026 (size=45) 2024-11-07T11:13:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741850_1026 (size=45) 2024-11-07T11:13:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741850_1026 (size=45) 2024-11-07T11:13:39,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7aa9c0d5fa702e40d1908a62541f05e4, disabling compactions & flushes 2024-11-07T11:13:39,706 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:39,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:39,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. after waiting 0 ms 2024-11-07T11:13:39,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:39,706 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:39,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7aa9c0d5fa702e40d1908a62541f05e4: Waiting for close lock at 1730978019706Disabling compacts and flushes for region at 1730978019706Disabling writes for close at 1730978019706Writing region close event to WAL at 1730978019706Closed at 1730978019706 2024-11-07T11:13:39,707 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 156088cd75fa40150a09828c74f7a0a1, NAME => 'test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741851_1027 (size=45) 2024-11-07T11:13:39,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741851_1027 (size=45) 2024-11-07T11:13:39,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741851_1027 (size=45) 2024-11-07T11:13:39,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1bffef86bb7015d7d1d3993e690e3a3c, disabling compactions & flushes 2024-11-07T11:13:39,714 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:39,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:39,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. after waiting 0 ms 2024-11-07T11:13:39,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:39,714 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:39,715 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1bffef86bb7015d7d1d3993e690e3a3c: Waiting for close lock at 1730978019714Disabling compacts and flushes for region at 1730978019714Disabling writes for close at 1730978019714Writing region close event to WAL at 1730978019714Closed at 1730978019714 2024-11-07T11:13:39,715 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c49878281e74ac31cf82a45639f5ef6c, NAME => 'test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741852_1028 (size=45) 2024-11-07T11:13:39,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741852_1028 (size=45) 2024-11-07T11:13:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741852_1028 (size=45) 2024-11-07T11:13:39,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 156088cd75fa40150a09828c74f7a0a1, disabling compactions & flushes 2024-11-07T11:13:39,721 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:39,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:39,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. after waiting 0 ms 2024-11-07T11:13:39,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:39,721 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:39,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 156088cd75fa40150a09828c74f7a0a1: Waiting for close lock at 1730978019721Disabling compacts and flushes for region at 1730978019721Disabling writes for close at 1730978019721Writing region close event to WAL at 1730978019721Closed at 1730978019721 2024-11-07T11:13:39,722 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 28bd8fcabc9d58f119d30748a4d3e741, NAME => 'test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741853_1029 (size=45) 2024-11-07T11:13:39,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741853_1029 (size=45) 2024-11-07T11:13:39,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741853_1029 (size=45) 2024-11-07T11:13:39,734 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,734 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c49878281e74ac31cf82a45639f5ef6c, disabling compactions & flushes 2024-11-07T11:13:39,734 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:39,734 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:39,734 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. after waiting 0 ms 2024-11-07T11:13:39,734 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:39,734 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:39,734 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c49878281e74ac31cf82a45639f5ef6c: Waiting for close lock at 1730978019734Disabling compacts and flushes for region at 1730978019734Disabling writes for close at 1730978019734Writing region close event to WAL at 1730978019734Closed at 1730978019734 2024-11-07T11:13:39,735 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54ad5e2c72f82a5eca4c14122252f4d4, NAME => 'test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741854_1030 (size=45) 2024-11-07T11:13:39,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741854_1030 (size=45) 2024-11-07T11:13:39,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741854_1030 (size=45) 2024-11-07T11:13:39,748 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,748 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 28bd8fcabc9d58f119d30748a4d3e741, disabling compactions & flushes 2024-11-07T11:13:39,748 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:39,749 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:39,749 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. after waiting 0 ms 2024-11-07T11:13:39,749 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:39,749 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:39,749 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 28bd8fcabc9d58f119d30748a4d3e741: Waiting for close lock at 1730978019748Disabling compacts and flushes for region at 1730978019748Disabling writes for close at 1730978019749 (+1 ms)Writing region close event to WAL at 1730978019749Closed at 1730978019749 2024-11-07T11:13:39,750 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9397bd7c5b10fb3fe8f98f0911b2382b, NAME => 'test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741855_1031 (size=45) 2024-11-07T11:13:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741855_1031 (size=45) 2024-11-07T11:13:39,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741855_1031 (size=45) 2024-11-07T11:13:39,762 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,762 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 54ad5e2c72f82a5eca4c14122252f4d4, disabling compactions & flushes 2024-11-07T11:13:39,762 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:39,762 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:39,762 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. after waiting 0 ms 2024-11-07T11:13:39,762 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:39,762 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:39,762 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54ad5e2c72f82a5eca4c14122252f4d4: Waiting for close lock at 1730978019762Disabling compacts and flushes for region at 1730978019762Disabling writes for close at 1730978019762Writing region close event to WAL at 1730978019762Closed at 1730978019762 2024-11-07T11:13:39,763 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1595aa61b66e39607e5d88bddc7e92d2, NAME => 'test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741856_1032 (size=45) 2024-11-07T11:13:39,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741856_1032 (size=45) 2024-11-07T11:13:39,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741856_1032 (size=45) 2024-11-07T11:13:39,769 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,769 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9397bd7c5b10fb3fe8f98f0911b2382b, disabling compactions & flushes 2024-11-07T11:13:39,769 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:39,769 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:39,769 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. after waiting 0 ms 2024-11-07T11:13:39,769 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:39,769 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:39,769 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9397bd7c5b10fb3fe8f98f0911b2382b: Waiting for close lock at 1730978019769Disabling compacts and flushes for region at 1730978019769Disabling writes for close at 1730978019769Writing region close event to WAL at 1730978019769Closed at 1730978019769 2024-11-07T11:13:39,770 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1aa45a9eec932f6b88e14ccca0f48aae, NAME => 'test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741857_1033 (size=45) 2024-11-07T11:13:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741857_1033 (size=45) 2024-11-07T11:13:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741857_1033 (size=45) 2024-11-07T11:13:39,781 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,781 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1595aa61b66e39607e5d88bddc7e92d2, disabling compactions & flushes 2024-11-07T11:13:39,781 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:39,781 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:39,781 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. after waiting 0 ms 2024-11-07T11:13:39,781 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:39,781 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:39,781 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1595aa61b66e39607e5d88bddc7e92d2: Waiting for close lock at 1730978019781Disabling compacts and flushes for region at 1730978019781Disabling writes for close at 1730978019781Writing region close event to WAL at 1730978019781Closed at 1730978019781 2024-11-07T11:13:39,782 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 89edfc8d260883f410baf2ebd451540e, NAME => 'test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741858_1034 (size=45) 2024-11-07T11:13:39,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741858_1034 (size=45) 2024-11-07T11:13:39,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741858_1034 (size=45) 2024-11-07T11:13:39,788 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,788 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1aa45a9eec932f6b88e14ccca0f48aae, disabling compactions & flushes 2024-11-07T11:13:39,788 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:39,788 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:39,788 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. after waiting 0 ms 2024-11-07T11:13:39,788 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:39,788 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:39,788 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1aa45a9eec932f6b88e14ccca0f48aae: Waiting for close lock at 1730978019788Disabling compacts and flushes for region at 1730978019788Disabling writes for close at 1730978019788Writing region close event to WAL at 1730978019788Closed at 1730978019788 2024-11-07T11:13:39,789 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a32ca03dfb3b86b085738b7e4da532b9, NAME => 'test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741859_1035 (size=45) 2024-11-07T11:13:39,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741859_1035 (size=45) 2024-11-07T11:13:39,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741859_1035 (size=45) 2024-11-07T11:13:39,796 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,796 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 89edfc8d260883f410baf2ebd451540e, disabling compactions & flushes 2024-11-07T11:13:39,796 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:39,796 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:39,796 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. after waiting 0 ms 2024-11-07T11:13:39,796 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:39,796 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:39,796 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 89edfc8d260883f410baf2ebd451540e: Waiting for close lock at 1730978019796Disabling compacts and flushes for region at 1730978019796Disabling writes for close at 1730978019796Writing region close event to WAL at 1730978019796Closed at 1730978019796 2024-11-07T11:13:39,797 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dac17ebdf8c70ae5014b41449b83b0ae, NAME => 'test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741860_1036 (size=45) 2024-11-07T11:13:39,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741860_1036 (size=45) 2024-11-07T11:13:39,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741860_1036 (size=45) 2024-11-07T11:13:39,805 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,805 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a32ca03dfb3b86b085738b7e4da532b9, disabling compactions & flushes 2024-11-07T11:13:39,805 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:39,805 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:39,805 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. after waiting 0 ms 2024-11-07T11:13:39,805 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:39,805 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:39,805 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a32ca03dfb3b86b085738b7e4da532b9: Waiting for close lock at 1730978019805Disabling compacts and flushes for region at 1730978019805Disabling writes for close at 1730978019805Writing region close event to WAL at 1730978019805Closed at 1730978019805 2024-11-07T11:13:39,806 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 38ac69d6e0db8e90441ae19a5f2f45e8, NAME => 'test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741861_1037 (size=45) 2024-11-07T11:13:39,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741861_1037 (size=45) 2024-11-07T11:13:39,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741861_1037 (size=45) 2024-11-07T11:13:39,810 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,810 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dac17ebdf8c70ae5014b41449b83b0ae, disabling compactions & flushes 2024-11-07T11:13:39,810 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:39,811 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:39,811 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. after waiting 0 ms 2024-11-07T11:13:39,811 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:39,811 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:39,811 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dac17ebdf8c70ae5014b41449b83b0ae: Waiting for close lock at 1730978019810Disabling compacts and flushes for region at 1730978019810Disabling writes for close at 1730978019811 (+1 ms)Writing region close event to WAL at 1730978019811Closed at 1730978019811 2024-11-07T11:13:39,812 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 80ba69ee8a10fcba947d5f7bb1fa9b5b, NAME => 'test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab 2024-11-07T11:13:39,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741862_1038 (size=45) 2024-11-07T11:13:39,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741862_1038 (size=45) 2024-11-07T11:13:39,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741862_1038 (size=45) 2024-11-07T11:13:39,818 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,819 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 38ac69d6e0db8e90441ae19a5f2f45e8, disabling compactions & flushes 2024-11-07T11:13:39,819 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:39,819 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:39,819 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. after waiting 0 ms 2024-11-07T11:13:39,819 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:39,819 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:39,819 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 38ac69d6e0db8e90441ae19a5f2f45e8: Waiting for close lock at 1730978019818Disabling compacts and flushes for region at 1730978019818Disabling writes for close at 1730978019819 (+1 ms)Writing region close event to WAL at 1730978019819Closed at 1730978019819 2024-11-07T11:13:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741863_1039 (size=42) 2024-11-07T11:13:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741863_1039 (size=42) 2024-11-07T11:13:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741863_1039 (size=42) 2024-11-07T11:13:39,822 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:39,822 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 80ba69ee8a10fcba947d5f7bb1fa9b5b, disabling compactions & flushes 2024-11-07T11:13:39,822 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:39,822 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:39,822 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. after waiting 0 ms 2024-11-07T11:13:39,822 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:39,822 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:39,822 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 80ba69ee8a10fcba947d5f7bb1fa9b5b: Waiting for close lock at 1730978019822Disabling compacts and flushes for region at 1730978019822Disabling writes for close at 1730978019822Writing region close event to WAL at 1730978019822Closed at 1730978019822 2024-11-07T11:13:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:13:40,095 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,095 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 314baf86ed26932b48f8c4cef53c1bcc, disabling compactions & flushes 2024-11-07T11:13:40,095 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,095 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,095 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. after waiting 0 ms 2024-11-07T11:13:40,095 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,096 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,096 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 314baf86ed26932b48f8c4cef53c1bcc: Waiting for close lock at 1730978020095Disabling compacts and flushes for region at 1730978020095Disabling writes for close at 1730978020095Writing region close event to WAL at 1730978020096 (+1 ms)Closed at 1730978020096 2024-11-07T11:13:40,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T11:13:40,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1730978019494.880afe4596d2f9f36732bf50351d0096.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1730978019494.356e319324497934b3703225e899ce57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1730978020100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730978020100"}]},"ts":"1730978020100"} 2024-11-07T11:13:40,151 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-07T11:13:40,153 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T11:13:40,155 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730978020153"}]},"ts":"1730978020153"} 2024-11-07T11:13:40,159 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-07T11:13:40,162 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {4ab3d709c6df=0} racks are {/default-rack=0} 2024-11-07T11:13:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:13:40,179 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:13:40,180 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:13:40,180 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-07T11:13:40,180 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:13:40,180 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:13:40,180 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T11:13:40,180 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:13:40,180 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:13:40,180 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T11:13:40,180 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:13:40,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=908579c522ec42c02649f6f9625ec8a4, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebf8c75d5f559642dd7af8561a14d668, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e6670dae2fae722a1a46cdb36b5a703a, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d7939d9f92fe5210293a2217ff7af696, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=356e319324497934b3703225e899ce57, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8c16ac0e25710f923881fe49bf532cf, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6356f249da7a2a5ae2d1836b77641b9d, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78b9a3cb232752746916dce9a515cf81, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=314baf86ed26932b48f8c4cef53c1bcc, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1bffef86bb7015d7d1d3993e690e3a3c, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=156088cd75fa40150a09828c74f7a0a1, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=28bd8fcabc9d58f119d30748a4d3e741, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=54ad5e2c72f82a5eca4c14122252f4d4, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1595aa61b66e39607e5d88bddc7e92d2, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1aa45a9eec932f6b88e14ccca0f48aae, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a32ca03dfb3b86b085738b7e4da532b9, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dac17ebdf8c70ae5014b41449b83b0ae, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80ba69ee8a10fcba947d5f7bb1fa9b5b, ASSIGN}] 2024-11-07T11:13:40,191 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80ba69ee8a10fcba947d5f7bb1fa9b5b, ASSIGN 2024-11-07T11:13:40,193 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=80ba69ee8a10fcba947d5f7bb1fa9b5b, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:40,195 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,196 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dac17ebdf8c70ae5014b41449b83b0ae, ASSIGN 2024-11-07T11:13:40,198 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dac17ebdf8c70ae5014b41449b83b0ae, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,199 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a32ca03dfb3b86b085738b7e4da532b9, ASSIGN 2024-11-07T11:13:40,200 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a32ca03dfb3b86b085738b7e4da532b9, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,202 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:40,203 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,204 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1aa45a9eec932f6b88e14ccca0f48aae, ASSIGN 2024-11-07T11:13:40,205 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1aa45a9eec932f6b88e14ccca0f48aae, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,206 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1595aa61b66e39607e5d88bddc7e92d2, ASSIGN 2024-11-07T11:13:40,208 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1595aa61b66e39607e5d88bddc7e92d2, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,209 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:40,210 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,211 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=54ad5e2c72f82a5eca4c14122252f4d4, ASSIGN 2024-11-07T11:13:40,212 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=54ad5e2c72f82a5eca4c14122252f4d4, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,214 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=28bd8fcabc9d58f119d30748a4d3e741, ASSIGN 2024-11-07T11:13:40,215 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=28bd8fcabc9d58f119d30748a4d3e741, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,216 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:40,217 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,219 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=156088cd75fa40150a09828c74f7a0a1, ASSIGN 2024-11-07T11:13:40,220 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=156088cd75fa40150a09828c74f7a0a1, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,221 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1bffef86bb7015d7d1d3993e690e3a3c, ASSIGN 2024-11-07T11:13:40,222 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1bffef86bb7015d7d1d3993e690e3a3c, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,224 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:40,225 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,226 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=314baf86ed26932b48f8c4cef53c1bcc, ASSIGN 2024-11-07T11:13:40,227 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=314baf86ed26932b48f8c4cef53c1bcc, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,229 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78b9a3cb232752746916dce9a515cf81, ASSIGN 2024-11-07T11:13:40,230 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=78b9a3cb232752746916dce9a515cf81, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,231 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6356f249da7a2a5ae2d1836b77641b9d, ASSIGN 2024-11-07T11:13:40,232 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6356f249da7a2a5ae2d1836b77641b9d, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,233 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8c16ac0e25710f923881fe49bf532cf, ASSIGN 2024-11-07T11:13:40,234 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f8c16ac0e25710f923881fe49bf532cf, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,236 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:40,237 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=356e319324497934b3703225e899ce57, ASSIGN 2024-11-07T11:13:40,239 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=356e319324497934b3703225e899ce57, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,240 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:40,241 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,242 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d7939d9f92fe5210293a2217ff7af696, ASSIGN 2024-11-07T11:13:40,243 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d7939d9f92fe5210293a2217ff7af696, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,245 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e6670dae2fae722a1a46cdb36b5a703a, ASSIGN 2024-11-07T11:13:40,246 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e6670dae2fae722a1a46cdb36b5a703a, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,247 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ebf8c75d5f559642dd7af8561a14d668, ASSIGN 2024-11-07T11:13:40,248 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ebf8c75d5f559642dd7af8561a14d668, ASSIGN; state=OFFLINE, location=4ab3d709c6df,33951,1730978016967; forceNewPlan=false, retain=false 2024-11-07T11:13:40,250 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=908579c522ec42c02649f6f9625ec8a4, ASSIGN 2024-11-07T11:13:40,251 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=908579c522ec42c02649f6f9625ec8a4, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43401,1730978016916; forceNewPlan=false, retain=false 2024-11-07T11:13:40,252 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:40,254 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN; state=OFFLINE, location=4ab3d709c6df,43027,1730978016820; forceNewPlan=false, retain=false 2024-11-07T11:13:40,350 INFO [4ab3d709c6df:37891 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-07T11:13:40,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=80ba69ee8a10fcba947d5f7bb1fa9b5b, ASSIGN because future has completed 2024-11-07T11:13:40,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80ba69ee8a10fcba947d5f7bb1fa9b5b, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a32ca03dfb3b86b085738b7e4da532b9, ASSIGN because future has completed 2024-11-07T11:13:40,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1595aa61b66e39607e5d88bddc7e92d2, ASSIGN because future has completed 2024-11-07T11:13:40,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=28bd8fcabc9d58f119d30748a4d3e741, ASSIGN because future has completed 2024-11-07T11:13:40,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28bd8fcabc9d58f119d30748a4d3e741, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1bffef86bb7015d7d1d3993e690e3a3c, ASSIGN because future has completed 2024-11-07T11:13:40,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6356f249da7a2a5ae2d1836b77641b9d, ASSIGN because future has completed 2024-11-07T11:13:40,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6356f249da7a2a5ae2d1836b77641b9d, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bffef86bb7015d7d1d3993e690e3a3c, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1595aa61b66e39607e5d88bddc7e92d2, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a32ca03dfb3b86b085738b7e4da532b9, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=356e319324497934b3703225e899ce57, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=908579c522ec42c02649f6f9625ec8a4, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=356e319324497934b3703225e899ce57, ASSIGN because future has completed 2024-11-07T11:13:40,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 356e319324497934b3703225e899ce57, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e6670dae2fae722a1a46cdb36b5a703a, ASSIGN because future has completed 2024-11-07T11:13:40,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=908579c522ec42c02649f6f9625ec8a4, ASSIGN because future has completed 2024-11-07T11:13:40,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN because future has completed 2024-11-07T11:13:40,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN because future has completed 2024-11-07T11:13:40,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN because future has completed 2024-11-07T11:13:40,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN because future has completed 2024-11-07T11:13:40,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN because future has completed 2024-11-07T11:13:40,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 908579c522ec42c02649f6f9625ec8a4, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6670dae2fae722a1a46cdb36b5a703a, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:40,439 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,440 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a5b587e52af29a25529015bea97baa17, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,440 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,441 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,441 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=156088cd75fa40150a09828c74f7a0a1, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN because future has completed 2024-11-07T11:13:40,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN because future has completed 2024-11-07T11:13:40,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN because future has completed 2024-11-07T11:13:40,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dac17ebdf8c70ae5014b41449b83b0ae, ASSIGN because future has completed 2024-11-07T11:13:40,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure dac17ebdf8c70ae5014b41449b83b0ae, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1aa45a9eec932f6b88e14ccca0f48aae, ASSIGN because future has completed 2024-11-07T11:13:40,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=54ad5e2c72f82a5eca4c14122252f4d4, ASSIGN because future has completed 2024-11-07T11:13:40,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=156088cd75fa40150a09828c74f7a0a1, ASSIGN because future has completed 2024-11-07T11:13:40,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 156088cd75fa40150a09828c74f7a0a1, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=314baf86ed26932b48f8c4cef53c1bcc, ASSIGN because future has completed 2024-11-07T11:13:40,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 314baf86ed26932b48f8c4cef53c1bcc, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ad5e2c72f82a5eca4c14122252f4d4, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aa45a9eec932f6b88e14ccca0f48aae, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43027,1730978016820}] 2024-11-07T11:13:40,481 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=78b9a3cb232752746916dce9a515cf81, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f8c16ac0e25710f923881fe49bf532cf, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=d7939d9f92fe5210293a2217ff7af696, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ebf8c75d5f559642dd7af8561a14d668, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=78b9a3cb232752746916dce9a515cf81, ASSIGN because future has completed 2024-11-07T11:13:40,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78b9a3cb232752746916dce9a515cf81, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f8c16ac0e25710f923881fe49bf532cf, ASSIGN because future has completed 2024-11-07T11:13:40,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d7939d9f92fe5210293a2217ff7af696, ASSIGN because future has completed 2024-11-07T11:13:40,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ebf8c75d5f559642dd7af8561a14d668, ASSIGN because future has completed 2024-11-07T11:13:40,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebf8c75d5f559642dd7af8561a14d668, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7939d9f92fe5210293a2217ff7af696, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8c16ac0e25710f923881fe49bf532cf, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:40,515 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:13:40,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54077, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:13:40,524 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:40,525 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => a32ca03dfb3b86b085738b7e4da532b9, NAME => 'test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-07T11:13:40,525 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,525 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,526 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,526 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,529 INFO [StoreOpener-a32ca03dfb3b86b085738b7e4da532b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,531 INFO [StoreOpener-a32ca03dfb3b86b085738b7e4da532b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a32ca03dfb3b86b085738b7e4da532b9 columnFamilyName family 2024-11-07T11:13:40,531 DEBUG [StoreOpener-a32ca03dfb3b86b085738b7e4da532b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,533 INFO [StoreOpener-a32ca03dfb3b86b085738b7e4da532b9-1 {}] regionserver.HStore(327): Store=a32ca03dfb3b86b085738b7e4da532b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,533 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,534 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,534 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,535 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,535 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,537 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,540 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a32ca03dfb3b86b085738b7e4da532b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,541 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened a32ca03dfb3b86b085738b7e4da532b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73311705, jitterRate=0.09242953360080719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,541 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:40,542 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for a32ca03dfb3b86b085738b7e4da532b9: Running coprocessor pre-open hook at 1730978020526Writing region info on filesystem at 1730978020526Initializing all the Stores at 1730978020528 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020528Cleaning up temporary data from old regions at 1730978020535 (+7 ms)Running coprocessor post-open hooks at 1730978020541 (+6 ms)Region opened successfully at 1730978020542 (+1 ms) 2024-11-07T11:13:40,544 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9., pid=36, masterSystemTime=1730978020514 2024-11-07T11:13:40,547 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:40,547 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:40,548 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:40,548 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 6356f249da7a2a5ae2d1836b77641b9d, NAME => 'test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-07T11:13:40,548 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,548 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,548 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,549 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,549 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,551 INFO [StoreOpener-6356f249da7a2a5ae2d1836b77641b9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a32ca03dfb3b86b085738b7e4da532b9, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,554 INFO [StoreOpener-6356f249da7a2a5ae2d1836b77641b9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6356f249da7a2a5ae2d1836b77641b9d columnFamilyName family 2024-11-07T11:13:40,554 DEBUG [StoreOpener-6356f249da7a2a5ae2d1836b77641b9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,555 INFO [StoreOpener-6356f249da7a2a5ae2d1836b77641b9d-1 {}] regionserver.HStore(327): Store=6356f249da7a2a5ae2d1836b77641b9d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,555 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,557 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,558 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=27 2024-11-07T11:13:40,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure a32ca03dfb3b86b085738b7e4da532b9, server=4ab3d709c6df,43401,1730978016916 in 166 msec 2024-11-07T11:13:40,559 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,560 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a32ca03dfb3b86b085738b7e4da532b9, ASSIGN in 376 msec 2024-11-07T11:13:40,563 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:40,563 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 38ac69d6e0db8e90441ae19a5f2f45e8, NAME => 'test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-07T11:13:40,564 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,564 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,564 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,564 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,565 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,566 INFO [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,568 INFO [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38ac69d6e0db8e90441ae19a5f2f45e8 columnFamilyName family 2024-11-07T11:13:40,568 DEBUG [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,568 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/6356f249da7a2a5ae2d1836b77641b9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,569 INFO [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] regionserver.HStore(327): Store=38ac69d6e0db8e90441ae19a5f2f45e8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,569 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 6356f249da7a2a5ae2d1836b77641b9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60882644, jitterRate=-0.09277790784835815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,569 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:40,569 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,569 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 6356f249da7a2a5ae2d1836b77641b9d: Running coprocessor pre-open hook at 1730978020549Writing region info on filesystem at 1730978020549Initializing all the Stores at 1730978020551 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020551Cleaning up temporary data from old regions at 1730978020560 (+9 ms)Running coprocessor post-open hooks at 1730978020569 (+9 ms)Region opened successfully at 1730978020569 2024-11-07T11:13:40,570 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,570 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d., pid=33, masterSystemTime=1730978020514 2024-11-07T11:13:40,571 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,571 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,572 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,574 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:40,574 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:40,575 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,575 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:40,575 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,575 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 908579c522ec42c02649f6f9625ec8a4, NAME => 'test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-07T11:13:40,575 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,575 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,576 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,576 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,578 INFO [StoreOpener-908579c522ec42c02649f6f9625ec8a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,580 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/38ac69d6e0db8e90441ae19a5f2f45e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,580 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 38ac69d6e0db8e90441ae19a5f2f45e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70071866, jitterRate=0.04415217041969299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,581 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:40,581 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 38ac69d6e0db8e90441ae19a5f2f45e8: Running coprocessor pre-open hook at 1730978020564Writing region info on filesystem at 1730978020564Initializing all the Stores at 1730978020566 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020566Cleaning up temporary data from old regions at 1730978020572 (+6 ms)Running coprocessor post-open hooks at 1730978020581 (+9 ms)Region opened successfully at 1730978020581 2024-11-07T11:13:40,581 INFO [StoreOpener-908579c522ec42c02649f6f9625ec8a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 908579c522ec42c02649f6f9625ec8a4 columnFamilyName family 2024-11-07T11:13:40,582 DEBUG [StoreOpener-908579c522ec42c02649f6f9625ec8a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,582 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8., pid=38, masterSystemTime=1730978020558 2024-11-07T11:13:40,583 INFO [StoreOpener-908579c522ec42c02649f6f9625ec8a4-1 {}] regionserver.HStore(327): Store=908579c522ec42c02649f6f9625ec8a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,583 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,584 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,586 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:40,586 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:40,587 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => c49878281e74ac31cf82a45639f5ef6c, NAME => 'test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,587 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,588 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6356f249da7a2a5ae2d1836b77641b9d, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,590 INFO [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,591 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,593 INFO [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49878281e74ac31cf82a45639f5ef6c columnFamilyName family 2024-11-07T11:13:40,593 DEBUG [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,594 INFO [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] regionserver.HStore(327): Store=c49878281e74ac31cf82a45639f5ef6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,595 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,597 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,598 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=14 2024-11-07T11:13:40,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 6356f249da7a2a5ae2d1836b77641b9d, server=4ab3d709c6df,43401,1730978016916 in 223 msec 2024-11-07T11:13:40,600 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,600 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,602 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:13:40,605 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/908579c522ec42c02649f6f9625ec8a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48229, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:13:40,605 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6356f249da7a2a5ae2d1836b77641b9d, ASSIGN in 417 msec 2024-11-07T11:13:40,608 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 908579c522ec42c02649f6f9625ec8a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65026552, jitterRate=-0.0310288667678833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,608 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:40,608 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 908579c522ec42c02649f6f9625ec8a4: Running coprocessor pre-open hook at 1730978020576Writing region info on filesystem at 1730978020576Initializing all the Stores at 1730978020578 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020578Cleaning up temporary data from old regions at 1730978020587 (+9 ms)Running coprocessor post-open hooks at 1730978020608 (+21 ms)Region opened successfully at 1730978020608 2024-11-07T11:13:40,611 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4., pid=43, masterSystemTime=1730978020514 2024-11-07T11:13:40,614 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:40,615 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:40,615 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:40,615 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 80ba69ee8a10fcba947d5f7bb1fa9b5b, NAME => 'test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-07T11:13:40,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=29 2024-11-07T11:13:40,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43027,1730978016820 in 208 msec 2024-11-07T11:13:40,616 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,616 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,616 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,616 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,618 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:40,618 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => dac17ebdf8c70ae5014b41449b83b0ae, NAME => 'test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-07T11:13:40,619 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,619 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,619 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,619 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN in 433 msec 2024-11-07T11:13:40,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,625 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/c49878281e74ac31cf82a45639f5ef6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,625 INFO [StoreOpener-80ba69ee8a10fcba947d5f7bb1fa9b5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 908579c522ec42c02649f6f9625ec8a4, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,627 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened c49878281e74ac31cf82a45639f5ef6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65462589, jitterRate=-0.02453140914440155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,627 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:40,628 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for c49878281e74ac31cf82a45639f5ef6c: Running coprocessor pre-open hook at 1730978020588Writing region info on filesystem at 1730978020588Initializing all the Stores at 1730978020589 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020590 (+1 ms)Cleaning up temporary data from old regions at 1730978020600 (+10 ms)Running coprocessor post-open hooks at 1730978020627 (+27 ms)Region opened successfully at 1730978020627 2024-11-07T11:13:40,628 INFO [StoreOpener-80ba69ee8a10fcba947d5f7bb1fa9b5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80ba69ee8a10fcba947d5f7bb1fa9b5b columnFamilyName family 2024-11-07T11:13:40,629 DEBUG [StoreOpener-80ba69ee8a10fcba947d5f7bb1fa9b5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,629 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c., pid=39, masterSystemTime=1730978020558 2024-11-07T11:13:40,630 INFO [StoreOpener-80ba69ee8a10fcba947d5f7bb1fa9b5b-1 {}] regionserver.HStore(327): Store=80ba69ee8a10fcba947d5f7bb1fa9b5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,630 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,632 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,633 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,634 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,634 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,634 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:40,634 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:40,634 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:40,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=6 2024-11-07T11:13:40,635 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 89edfc8d260883f410baf2ebd451540e, NAME => 'test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-07T11:13:40,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 908579c522ec42c02649f6f9625ec8a4, server=4ab3d709c6df,43401,1730978016916 in 198 msec 2024-11-07T11:13:40,635 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,635 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,635 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,635 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,637 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=908579c522ec42c02649f6f9625ec8a4, ASSIGN in 452 msec 2024-11-07T11:13:40,640 INFO [StoreOpener-dac17ebdf8c70ae5014b41449b83b0ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c49878281e74ac31cf82a45639f5ef6c, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,643 INFO [StoreOpener-dac17ebdf8c70ae5014b41449b83b0ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dac17ebdf8c70ae5014b41449b83b0ae columnFamilyName family 2024-11-07T11:13:40,643 DEBUG [StoreOpener-dac17ebdf8c70ae5014b41449b83b0ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,644 INFO [StoreOpener-dac17ebdf8c70ae5014b41449b83b0ae-1 {}] regionserver.HStore(327): Store=dac17ebdf8c70ae5014b41449b83b0ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,645 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,646 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,647 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,648 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,648 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,648 INFO [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,650 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/80ba69ee8a10fcba947d5f7bb1fa9b5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,651 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 80ba69ee8a10fcba947d5f7bb1fa9b5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63872531, jitterRate=-0.04822511970996857}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,651 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:40,651 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 80ba69ee8a10fcba947d5f7bb1fa9b5b: Running coprocessor pre-open hook at 1730978020616Writing region info on filesystem at 1730978020616Initializing all the Stores at 1730978020618 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020618Cleaning up temporary data from old regions at 1730978020634 (+16 ms)Running coprocessor post-open hooks at 1730978020651 (+17 ms)Region opened successfully at 1730978020651 2024-11-07T11:13:40,652 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,653 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b., pid=31, masterSystemTime=1730978020514 2024-11-07T11:13:40,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=20 2024-11-07T11:13:40,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,43027,1730978016820 in 238 msec 2024-11-07T11:13:40,653 INFO [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89edfc8d260883f410baf2ebd451540e columnFamilyName family 2024-11-07T11:13:40,654 DEBUG [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,655 INFO [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] regionserver.HStore(327): Store=89edfc8d260883f410baf2ebd451540e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,655 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,657 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:40,657 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:40,657 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,657 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/dac17ebdf8c70ae5014b41449b83b0ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,657 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:40,657 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bffef86bb7015d7d1d3993e690e3a3c, NAME => 'test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,658 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened dac17ebdf8c70ae5014b41449b83b0ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74521296, jitterRate=0.11045384407043457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,658 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for dac17ebdf8c70ae5014b41449b83b0ae: Running coprocessor pre-open hook at 1730978020619Writing region info on filesystem at 1730978020620 (+1 ms)Initializing all the Stores at 1730978020621 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020621Cleaning up temporary data from old regions at 1730978020648 (+27 ms)Running coprocessor post-open hooks at 1730978020658 (+10 ms)Region opened successfully at 1730978020658 2024-11-07T11:13:40,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN in 470 msec 2024-11-07T11:13:40,659 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,659 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,661 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae., pid=46, masterSystemTime=1730978020602 2024-11-07T11:13:40,662 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,666 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80ba69ee8a10fcba947d5f7bb1fa9b5b, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,667 INFO [StoreOpener-1bffef86bb7015d7d1d3993e690e3a3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,669 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=test, region=80ba69ee8a10fcba947d5f7bb1fa9b5b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T11:13:40,669 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:40,669 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:40,669 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:40,670 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 78b9a3cb232752746916dce9a515cf81, NAME => 'test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-07T11:13:40,670 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,670 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,670 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,670 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,671 INFO [StoreOpener-1bffef86bb7015d7d1d3993e690e3a3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bffef86bb7015d7d1d3993e690e3a3c columnFamilyName family 2024-11-07T11:13:40,671 DEBUG [StoreOpener-1bffef86bb7015d7d1d3993e690e3a3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,672 INFO [StoreOpener-1bffef86bb7015d7d1d3993e690e3a3c-1 {}] regionserver.HStore(327): Store=1bffef86bb7015d7d1d3993e690e3a3c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,672 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,673 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,674 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,675 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,675 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure dac17ebdf8c70ae5014b41449b83b0ae, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,680 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,680 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/89edfc8d260883f410baf2ebd451540e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,681 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 89edfc8d260883f410baf2ebd451540e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61850045, jitterRate=-0.07836250960826874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,681 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:40,682 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 89edfc8d260883f410baf2ebd451540e: Running coprocessor pre-open hook at 1730978020636Writing region info on filesystem at 1730978020636Initializing all the Stores at 1730978020637 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020637Cleaning up temporary data from old regions at 1730978020659 (+22 ms)Running coprocessor post-open hooks at 1730978020681 (+22 ms)Region opened successfully at 1730978020682 (+1 ms) 2024-11-07T11:13:40,683 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e., pid=42, masterSystemTime=1730978020558 2024-11-07T11:13:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:13:40,688 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:40,688 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:40,689 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:40,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T11:13:40,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 80ba69ee8a10fcba947d5f7bb1fa9b5b, server=4ab3d709c6df,43401,1730978016916 in 322 msec 2024-11-07T11:13:40,689 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => ba16fdeba546ef71df7c8b7d75e0fc39, NAME => 'test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-07T11:13:40,689 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,689 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,689 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,690 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=80ba69ee8a10fcba947d5f7bb1fa9b5b, ASSIGN in 506 msec 2024-11-07T11:13:40,694 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=89edfc8d260883f410baf2ebd451540e, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,700 INFO [StoreOpener-78b9a3cb232752746916dce9a515cf81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,703 INFO [StoreOpener-78b9a3cb232752746916dce9a515cf81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78b9a3cb232752746916dce9a515cf81 columnFamilyName family 2024-11-07T11:13:40,703 DEBUG [StoreOpener-78b9a3cb232752746916dce9a515cf81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,704 INFO [StoreOpener-78b9a3cb232752746916dce9a515cf81-1 {}] regionserver.HStore(327): Store=78b9a3cb232752746916dce9a515cf81/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=28 2024-11-07T11:13:40,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure dac17ebdf8c70ae5014b41449b83b0ae, server=4ab3d709c6df,33951,1730978016967 in 250 msec 2024-11-07T11:13:40,704 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,706 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,707 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,708 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,708 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dac17ebdf8c70ae5014b41449b83b0ae, ASSIGN in 522 msec 2024-11-07T11:13:40,711 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,717 INFO [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=26 2024-11-07T11:13:40,719 INFO [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba16fdeba546ef71df7c8b7d75e0fc39 columnFamilyName family 2024-11-07T11:13:40,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,43027,1730978016820 in 286 msec 2024-11-07T11:13:40,719 DEBUG [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,721 INFO [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] regionserver.HStore(327): Store=ba16fdeba546ef71df7c8b7d75e0fc39/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,721 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,722 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,723 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN in 537 msec 2024-11-07T11:13:40,724 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,724 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,727 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,728 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1bffef86bb7015d7d1d3993e690e3a3c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,729 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 1bffef86bb7015d7d1d3993e690e3a3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64988753, jitterRate=-0.03159211575984955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,729 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:40,729 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 1bffef86bb7015d7d1d3993e690e3a3c: Running coprocessor pre-open hook at 1730978020659Writing region info on filesystem at 1730978020659Initializing all the Stores at 1730978020667 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020667Cleaning up temporary data from old regions at 1730978020675 (+8 ms)Running coprocessor post-open hooks at 1730978020729 (+54 ms)Region opened successfully at 1730978020729 2024-11-07T11:13:40,730 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ba16fdeba546ef71df7c8b7d75e0fc39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,731 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/78b9a3cb232752746916dce9a515cf81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,731 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c., pid=34, masterSystemTime=1730978020514 2024-11-07T11:13:40,731 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened ba16fdeba546ef71df7c8b7d75e0fc39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59826677, jitterRate=-0.10851304233074188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,732 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 78b9a3cb232752746916dce9a515cf81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68808104, jitterRate=0.02532064914703369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,732 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:40,732 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:40,732 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for ba16fdeba546ef71df7c8b7d75e0fc39: Running coprocessor pre-open hook at 1730978020690Writing region info on filesystem at 1730978020690Initializing all the Stores at 1730978020691 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020691Cleaning up temporary data from old regions at 1730978020724 (+33 ms)Running coprocessor post-open hooks at 1730978020732 (+8 ms)Region opened successfully at 1730978020732 2024-11-07T11:13:40,732 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 78b9a3cb232752746916dce9a515cf81: Running coprocessor pre-open hook at 1730978020671Writing region info on filesystem at 1730978020671Initializing all the Stores at 1730978020672 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020672Cleaning up temporary data from old regions at 1730978020708 (+36 ms)Running coprocessor post-open hooks at 1730978020732 (+24 ms)Region opened successfully at 1730978020732 2024-11-07T11:13:40,733 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39., pid=45, masterSystemTime=1730978020558 2024-11-07T11:13:40,734 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81., pid=53, masterSystemTime=1730978020602 2024-11-07T11:13:40,736 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:40,736 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:40,737 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:40,737 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595aa61b66e39607e5d88bddc7e92d2, NAME => 'test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-07T11:13:40,737 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,737 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,737 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,738 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,739 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,740 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:40,740 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:40,740 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:40,740 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 880afe4596d2f9f36732bf50351d0096, NAME => 'test,,1730978019494.880afe4596d2f9f36732bf50351d0096.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-07T11:13:40,741 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,741 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,,1730978019494.880afe4596d2f9f36732bf50351d0096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,741 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,741 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,742 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:40,742 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:40,742 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:40,743 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => ebf8c75d5f559642dd7af8561a14d668, NAME => 'test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-07T11:13:40,743 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,743 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,743 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,744 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bffef86bb7015d7d1d3993e690e3a3c, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,748 INFO [StoreOpener-1595aa61b66e39607e5d88bddc7e92d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,752 INFO [StoreOpener-1595aa61b66e39607e5d88bddc7e92d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595aa61b66e39607e5d88bddc7e92d2 columnFamilyName family 2024-11-07T11:13:40,752 DEBUG [StoreOpener-1595aa61b66e39607e5d88bddc7e92d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78b9a3cb232752746916dce9a515cf81, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,756 INFO [StoreOpener-1595aa61b66e39607e5d88bddc7e92d2-1 {}] regionserver.HStore(327): Store=1595aa61b66e39607e5d88bddc7e92d2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,756 INFO [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,756 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,758 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,759 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,760 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,760 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,760 INFO [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880afe4596d2f9f36732bf50351d0096 columnFamilyName family 2024-11-07T11:13:40,760 DEBUG [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=18 2024-11-07T11:13:40,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 1bffef86bb7015d7d1d3993e690e3a3c, server=4ab3d709c6df,43401,1730978016916 in 376 msec 2024-11-07T11:13:40,761 INFO [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] regionserver.HStore(327): Store=880afe4596d2f9f36732bf50351d0096/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,762 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,763 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,764 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,764 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1bffef86bb7015d7d1d3993e690e3a3c, ASSIGN in 578 msec 2024-11-07T11:13:40,765 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,765 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,768 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=4ab3d709c6df,33951,1730978016967, table=test, region=78b9a3cb232752746916dce9a515cf81. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T11:13:40,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,777 INFO [StoreOpener-ebf8c75d5f559642dd7af8561a14d668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=15 2024-11-07T11:13:40,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 78b9a3cb232752746916dce9a515cf81, server=4ab3d709c6df,33951,1730978016967 in 284 msec 2024-11-07T11:13:40,780 INFO [StoreOpener-ebf8c75d5f559642dd7af8561a14d668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebf8c75d5f559642dd7af8561a14d668 columnFamilyName family 2024-11-07T11:13:40,780 DEBUG [StoreOpener-ebf8c75d5f559642dd7af8561a14d668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,781 INFO [StoreOpener-ebf8c75d5f559642dd7af8561a14d668-1 {}] regionserver.HStore(327): Store=ebf8c75d5f559642dd7af8561a14d668/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,781 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,783 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=78b9a3cb232752746916dce9a515cf81, ASSIGN in 595 msec 2024-11-07T11:13:40,783 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,784 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,784 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,788 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,790 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1595aa61b66e39607e5d88bddc7e92d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,790 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/880afe4596d2f9f36732bf50351d0096/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,792 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1595aa61b66e39607e5d88bddc7e92d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61279135, jitterRate=-0.0868697315454483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,792 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:40,792 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1595aa61b66e39607e5d88bddc7e92d2: Running coprocessor pre-open hook at 1730978020738Writing region info on filesystem at 1730978020738Initializing all the Stores at 1730978020740 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020740Cleaning up temporary data from old regions at 1730978020760 (+20 ms)Running coprocessor post-open hooks at 1730978020792 (+32 ms)Region opened successfully at 1730978020792 2024-11-07T11:13:40,793 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 880afe4596d2f9f36732bf50351d0096; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71243741, jitterRate=0.061614468693733215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,793 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:40,793 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 880afe4596d2f9f36732bf50351d0096: Running coprocessor pre-open hook at 1730978020741Writing region info on filesystem at 1730978020741Initializing all the Stores at 1730978020744 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020744Cleaning up temporary data from old regions at 1730978020765 (+21 ms)Running coprocessor post-open hooks at 1730978020793 (+28 ms)Region opened successfully at 1730978020793 2024-11-07T11:13:40,793 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2., pid=35, masterSystemTime=1730978020514 2024-11-07T11:13:40,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,795 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1730978019494.880afe4596d2f9f36732bf50351d0096., pid=51, masterSystemTime=1730978020558 2024-11-07T11:13:40,796 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:40,797 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:40,797 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:40,797 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 28bd8fcabc9d58f119d30748a4d3e741, NAME => 'test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-07T11:13:40,798 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,798 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,798 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,798 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,800 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:40,800 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:40,800 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:40,801 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aa9c0d5fa702e40d1908a62541f05e4, NAME => 'test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-07T11:13:40,801 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,801 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,802 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,802 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=12 2024-11-07T11:13:40,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,43027,1730978016820 in 355 msec 2024-11-07T11:13:40,809 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ebf8c75d5f559642dd7af8561a14d668/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN in 622 msec 2024-11-07T11:13:40,810 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened ebf8c75d5f559642dd7af8561a14d668; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75451265, jitterRate=0.12431146204471588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,811 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:40,811 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for ebf8c75d5f559642dd7af8561a14d668: Running coprocessor pre-open hook at 1730978020744Writing region info on filesystem at 1730978020744Initializing all the Stores at 1730978020745 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020745Cleaning up temporary data from old regions at 1730978020784 (+39 ms)Running coprocessor post-open hooks at 1730978020811 (+27 ms)Region opened successfully at 1730978020811 2024-11-07T11:13:40,812 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668., pid=54, masterSystemTime=1730978020602 2024-11-07T11:13:40,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=880afe4596d2f9f36732bf50351d0096, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,817 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:40,817 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:40,817 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:40,817 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 1aa45a9eec932f6b88e14ccca0f48aae, NAME => 'test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-07T11:13:40,818 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,818 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,818 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,818 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,819 INFO [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,820 INFO [StoreOpener-28bd8fcabc9d58f119d30748a4d3e741-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,821 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,822 INFO [StoreOpener-28bd8fcabc9d58f119d30748a4d3e741-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bd8fcabc9d58f119d30748a4d3e741 columnFamilyName family 2024-11-07T11:13:40,823 DEBUG [StoreOpener-28bd8fcabc9d58f119d30748a4d3e741-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,824 INFO [StoreOpener-28bd8fcabc9d58f119d30748a4d3e741-1 {}] regionserver.HStore(327): Store=28bd8fcabc9d58f119d30748a4d3e741/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,824 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,825 INFO [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa9c0d5fa702e40d1908a62541f05e4 columnFamilyName family 2024-11-07T11:13:40,825 DEBUG [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,826 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,826 INFO [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] regionserver.HStore(327): Store=7aa9c0d5fa702e40d1908a62541f05e4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,826 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,827 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebf8c75d5f559642dd7af8561a14d668, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,828 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,828 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,828 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,829 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,829 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,829 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,831 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,832 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=5 2024-11-07T11:13:40,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,43027,1730978016820 in 355 msec 2024-11-07T11:13:40,836 INFO [StoreOpener-1aa45a9eec932f6b88e14ccca0f48aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,838 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/28bd8fcabc9d58f119d30748a4d3e741/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,839 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 28bd8fcabc9d58f119d30748a4d3e741; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69230699, jitterRate=0.03161780536174774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,839 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:40,839 INFO [StoreOpener-1aa45a9eec932f6b88e14ccca0f48aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa45a9eec932f6b88e14ccca0f48aae columnFamilyName family 2024-11-07T11:13:40,839 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 28bd8fcabc9d58f119d30748a4d3e741: Running coprocessor pre-open hook at 1730978020798Writing region info on filesystem at 1730978020798Initializing all the Stores at 1730978020801 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020801Cleaning up temporary data from old regions at 1730978020828 (+27 ms)Running coprocessor post-open hooks at 1730978020839 (+11 ms)Region opened successfully at 1730978020839 2024-11-07T11:13:40,839 DEBUG [StoreOpener-1aa45a9eec932f6b88e14ccca0f48aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,840 INFO [StoreOpener-1aa45a9eec932f6b88e14ccca0f48aae-1 {}] regionserver.HStore(327): Store=1aa45a9eec932f6b88e14ccca0f48aae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN in 651 msec 2024-11-07T11:13:40,840 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,841 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741., pid=32, masterSystemTime=1730978020514 2024-11-07T11:13:40,844 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,845 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,845 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:40,845 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:40,845 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => e6670dae2fae722a1a46cdb36b5a703a, NAME => 'test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,846 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,849 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,853 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/7aa9c0d5fa702e40d1908a62541f05e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,854 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 7aa9c0d5fa702e40d1908a62541f05e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66831795, jitterRate=-0.004128649830818176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,854 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:40,855 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 7aa9c0d5fa702e40d1908a62541f05e4: Running coprocessor pre-open hook at 1730978020802Writing region info on filesystem at 1730978020802Initializing all the Stores at 1730978020805 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020805Cleaning up temporary data from old regions at 1730978020829 (+24 ms)Running coprocessor post-open hooks at 1730978020854 (+25 ms)Region opened successfully at 1730978020854 2024-11-07T11:13:40,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-11-07T11:13:40,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ebf8c75d5f559642dd7af8561a14d668, server=4ab3d709c6df,33951,1730978016967 in 355 msec 2024-11-07T11:13:40,856 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4., pid=40, masterSystemTime=1730978020558 2024-11-07T11:13:40,860 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:40,860 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:40,860 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:40,861 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => a5b587e52af29a25529015bea97baa17, NAME => 'test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-07T11:13:40,861 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,861 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,861 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,861 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ebf8c75d5f559642dd7af8561a14d668, ASSIGN in 674 msec 2024-11-07T11:13:40,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,864 INFO [StoreOpener-e6670dae2fae722a1a46cdb36b5a703a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,865 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1aa45a9eec932f6b88e14ccca0f48aae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,866 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 1aa45a9eec932f6b88e14ccca0f48aae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72447867, jitterRate=0.07955734431743622}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,866 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:40,866 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 1aa45a9eec932f6b88e14ccca0f48aae: Running coprocessor pre-open hook at 1730978020819Writing region info on filesystem at 1730978020819Initializing all the Stores at 1730978020821 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020821Cleaning up temporary data from old regions at 1730978020846 (+25 ms)Running coprocessor post-open hooks at 1730978020866 (+20 ms)Region opened successfully at 1730978020866 2024-11-07T11:13:40,868 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae., pid=50, masterSystemTime=1730978020602 2024-11-07T11:13:40,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,870 INFO [StoreOpener-e6670dae2fae722a1a46cdb36b5a703a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6670dae2fae722a1a46cdb36b5a703a columnFamilyName family 2024-11-07T11:13:40,870 DEBUG [StoreOpener-e6670dae2fae722a1a46cdb36b5a703a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,871 INFO [StoreOpener-e6670dae2fae722a1a46cdb36b5a703a-1 {}] regionserver.HStore(327): Store=e6670dae2fae722a1a46cdb36b5a703a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,871 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,872 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,873 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,873 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:40,873 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:40,874 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:40,874 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => d7939d9f92fe5210293a2217ff7af696, NAME => 'test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-07T11:13:40,874 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,874 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,875 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,875 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,876 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,876 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,882 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,883 INFO [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=17 2024-11-07T11:13:40,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43027,1730978016820 in 460 msec 2024-11-07T11:13:40,885 INFO [StoreOpener-d7939d9f92fe5210293a2217ff7af696-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,886 INFO [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5b587e52af29a25529015bea97baa17 columnFamilyName family 2024-11-07T11:13:40,886 DEBUG [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,887 INFO [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] regionserver.HStore(327): Store=a5b587e52af29a25529015bea97baa17/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,888 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN in 701 msec 2024-11-07T11:13:40,888 INFO [StoreOpener-d7939d9f92fe5210293a2217ff7af696-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7939d9f92fe5210293a2217ff7af696 columnFamilyName family 2024-11-07T11:13:40,888 DEBUG [StoreOpener-d7939d9f92fe5210293a2217ff7af696-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,889 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,890 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,891 INFO [StoreOpener-d7939d9f92fe5210293a2217ff7af696-1 {}] regionserver.HStore(327): Store=d7939d9f92fe5210293a2217ff7af696/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,891 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,892 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,892 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,894 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aa45a9eec932f6b88e14ccca0f48aae, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,894 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,895 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,895 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,895 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,898 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/e6670dae2fae722a1a46cdb36b5a703a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,899 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a5b587e52af29a25529015bea97baa17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,899 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,899 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened e6670dae2fae722a1a46cdb36b5a703a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70160604, jitterRate=0.045474469661712646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,899 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:40,899 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for e6670dae2fae722a1a46cdb36b5a703a: Running coprocessor pre-open hook at 1730978020847Writing region info on filesystem at 1730978020847Initializing all the Stores at 1730978020849 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020849Cleaning up temporary data from old regions at 1730978020876 (+27 ms)Running coprocessor post-open hooks at 1730978020899 (+23 ms)Region opened successfully at 1730978020899 2024-11-07T11:13:40,900 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened a5b587e52af29a25529015bea97baa17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68486788, jitterRate=0.020532667636871338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,900 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:40,901 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for a5b587e52af29a25529015bea97baa17: Running coprocessor pre-open hook at 1730978020861Writing region info on filesystem at 1730978020861Initializing all the Stores at 1730978020863 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020863Cleaning up temporary data from old regions at 1730978020892 (+29 ms)Running coprocessor post-open hooks at 1730978020900 (+8 ms)Region opened successfully at 1730978020900 2024-11-07T11:13:40,901 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a., pid=44, masterSystemTime=1730978020514 2024-11-07T11:13:40,902 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1730978019494.a5b587e52af29a25529015bea97baa17., pid=52, masterSystemTime=1730978020558 2024-11-07T11:13:40,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=25 2024-11-07T11:13:40,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 1aa45a9eec932f6b88e14ccca0f48aae, server=4ab3d709c6df,33951,1730978016967 in 426 msec 2024-11-07T11:13:40,904 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:40,904 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:40,904 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:40,904 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 356e319324497934b3703225e899ce57, NAME => 'test,ggg,1730978019494.356e319324497934b3703225e899ce57.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-07T11:13:40,905 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/d7939d9f92fe5210293a2217ff7af696/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,905 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,905 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ggg,1730978019494.356e319324497934b3703225e899ce57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,905 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,905 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,905 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened d7939d9f92fe5210293a2217ff7af696; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65845720, jitterRate=-0.018822312355041504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,906 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:40,906 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for d7939d9f92fe5210293a2217ff7af696: Running coprocessor pre-open hook at 1730978020875Writing region info on filesystem at 1730978020875Initializing all the Stores at 1730978020877 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020877Cleaning up temporary data from old regions at 1730978020895 (+18 ms)Running coprocessor post-open hooks at 1730978020906 (+11 ms)Region opened successfully at 1730978020906 2024-11-07T11:13:40,907 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:40,907 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:40,907 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696., pid=55, masterSystemTime=1730978020602 2024-11-07T11:13:40,907 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,907 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:40,907 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 9397bd7c5b10fb3fe8f98f0911b2382b, NAME => 'test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-07T11:13:40,907 INFO [StoreOpener-356e319324497934b3703225e899ce57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,908 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,908 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,908 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,908 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,911 INFO [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,911 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:40,911 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:40,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6670dae2fae722a1a46cdb36b5a703a, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,911 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,911 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 314baf86ed26932b48f8c4cef53c1bcc, NAME => 'test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-07T11:13:40,912 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,912 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,912 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,912 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,912 INFO [StoreOpener-356e319324497934b3703225e899ce57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 356e319324497934b3703225e899ce57 columnFamilyName family 2024-11-07T11:13:40,913 DEBUG [StoreOpener-356e319324497934b3703225e899ce57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,913 INFO [StoreOpener-356e319324497934b3703225e899ce57-1 {}] regionserver.HStore(327): Store=356e319324497934b3703225e899ce57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,914 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,914 INFO [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9397bd7c5b10fb3fe8f98f0911b2382b columnFamilyName family 2024-11-07T11:13:40,914 DEBUG [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,915 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/356e319324497934b3703225e899ce57 2024-11-07T11:13:40,915 INFO [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] regionserver.HStore(327): Store=9397bd7c5b10fb3fe8f98f0911b2382b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,915 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/356e319324497934b3703225e899ce57 2024-11-07T11:13:40,915 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,916 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,916 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,917 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7939d9f92fe5210293a2217ff7af696, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,918 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,918 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,919 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,919 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,923 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=8 2024-11-07T11:13:40,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e6670dae2fae722a1a46cdb36b5a703a, server=4ab3d709c6df,43401,1730978016916 in 484 msec 2024-11-07T11:13:40,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e6670dae2fae722a1a46cdb36b5a703a, ASSIGN in 740 msec 2024-11-07T11:13:40,928 INFO [StoreOpener-314baf86ed26932b48f8c4cef53c1bcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,930 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/9397bd7c5b10fb3fe8f98f0911b2382b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,930 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/356e319324497934b3703225e899ce57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,931 INFO [StoreOpener-314baf86ed26932b48f8c4cef53c1bcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 314baf86ed26932b48f8c4cef53c1bcc columnFamilyName family 2024-11-07T11:13:40,931 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 356e319324497934b3703225e899ce57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67840425, jitterRate=0.010901108384132385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,931 DEBUG [StoreOpener-314baf86ed26932b48f8c4cef53c1bcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,931 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 356e319324497934b3703225e899ce57 2024-11-07T11:13:40,931 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 356e319324497934b3703225e899ce57: Running coprocessor pre-open hook at 1730978020905Writing region info on filesystem at 1730978020905Initializing all the Stores at 1730978020906 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020906Cleaning up temporary data from old regions at 1730978020916 (+10 ms)Running coprocessor post-open hooks at 1730978020931 (+15 ms)Region opened successfully at 1730978020931 2024-11-07T11:13:40,931 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 9397bd7c5b10fb3fe8f98f0911b2382b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67165014, jitterRate=8.367002010345459E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,932 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:40,932 INFO [StoreOpener-314baf86ed26932b48f8c4cef53c1bcc-1 {}] regionserver.HStore(327): Store=314baf86ed26932b48f8c4cef53c1bcc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,932 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 9397bd7c5b10fb3fe8f98f0911b2382b: Running coprocessor pre-open hook at 1730978020908Writing region info on filesystem at 1730978020908Initializing all the Stores at 1730978020910 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020910Cleaning up temporary data from old regions at 1730978020919 (+9 ms)Running coprocessor post-open hooks at 1730978020932 (+13 ms)Region opened successfully at 1730978020932 2024-11-07T11:13:40,932 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-07T11:13:40,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure d7939d9f92fe5210293a2217ff7af696, server=4ab3d709c6df,33951,1730978016967 in 434 msec 2024-11-07T11:13:40,933 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,934 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1730978019494.356e319324497934b3703225e899ce57., pid=37, masterSystemTime=1730978020514 2024-11-07T11:13:40,934 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b., pid=41, masterSystemTime=1730978020558 2024-11-07T11:13:40,934 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,935 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,935 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d7939d9f92fe5210293a2217ff7af696, ASSIGN in 749 msec 2024-11-07T11:13:40,938 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:40,938 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:40,938 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=356e319324497934b3703225e899ce57, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:40,940 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:40,940 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:40,942 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/314baf86ed26932b48f8c4cef53c1bcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 356e319324497934b3703225e899ce57, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:40,943 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 314baf86ed26932b48f8c4cef53c1bcc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61243754, jitterRate=-0.08739694952964783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,943 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:40,943 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,943 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 314baf86ed26932b48f8c4cef53c1bcc: Running coprocessor pre-open hook at 1730978020912Writing region info on filesystem at 1730978020912Initializing all the Stores at 1730978020914 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020914Cleaning up temporary data from old regions at 1730978020935 (+21 ms)Running coprocessor post-open hooks at 1730978020943 (+8 ms)Region opened successfully at 1730978020943 2024-11-07T11:13:40,944 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc., pid=48, masterSystemTime=1730978020602 2024-11-07T11:13:40,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,947 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:40,947 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:40,948 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 156088cd75fa40150a09828c74f7a0a1, NAME => 'test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-07T11:13:40,948 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,948 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,948 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,948 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,950 INFO [StoreOpener-156088cd75fa40150a09828c74f7a0a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-07T11:13:40,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 356e319324497934b3703225e899ce57, server=4ab3d709c6df,43401,1730978016916 in 550 msec 2024-11-07T11:13:40,953 INFO [StoreOpener-156088cd75fa40150a09828c74f7a0a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 156088cd75fa40150a09828c74f7a0a1 columnFamilyName family 2024-11-07T11:13:40,953 DEBUG [StoreOpener-156088cd75fa40150a09828c74f7a0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=356e319324497934b3703225e899ce57, ASSIGN in 768 msec 2024-11-07T11:13:40,954 INFO [StoreOpener-156088cd75fa40150a09828c74f7a0a1-1 {}] regionserver.HStore(327): Store=156088cd75fa40150a09828c74f7a0a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,954 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,955 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,956 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,957 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,957 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=23 2024-11-07T11:13:40,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43027,1730978016820 in 535 msec 2024-11-07T11:13:40,959 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN in 775 msec 2024-11-07T11:13:40,961 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,963 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/156088cd75fa40150a09828c74f7a0a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,964 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 156088cd75fa40150a09828c74f7a0a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59453294, jitterRate=-0.1140768826007843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,964 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:40,964 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 156088cd75fa40150a09828c74f7a0a1: Running coprocessor pre-open hook at 1730978020949Writing region info on filesystem at 1730978020949Initializing all the Stores at 1730978020950 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020950Cleaning up temporary data from old regions at 1730978020957 (+7 ms)Running coprocessor post-open hooks at 1730978020964 (+7 ms)Region opened successfully at 1730978020964 2024-11-07T11:13:40,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 314baf86ed26932b48f8c4cef53c1bcc, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,965 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1., pid=47, masterSystemTime=1730978020602 2024-11-07T11:13:40,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a5b587e52af29a25529015bea97baa17, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:40,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43027,1730978016820 because future has completed 2024-11-07T11:13:40,968 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:40,968 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:40,968 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:40,969 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 54ad5e2c72f82a5eca4c14122252f4d4, NAME => 'test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-07T11:13:40,969 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,969 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,969 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,969 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=16 2024-11-07T11:13:40,972 INFO [StoreOpener-54ad5e2c72f82a5eca4c14122252f4d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 314baf86ed26932b48f8c4cef53c1bcc, server=4ab3d709c6df,33951,1730978016967 in 509 msec 2024-11-07T11:13:40,973 INFO [StoreOpener-54ad5e2c72f82a5eca4c14122252f4d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ad5e2c72f82a5eca4c14122252f4d4 columnFamilyName family 2024-11-07T11:13:40,974 DEBUG [StoreOpener-54ad5e2c72f82a5eca4c14122252f4d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=314baf86ed26932b48f8c4cef53c1bcc, ASSIGN in 789 msec 2024-11-07T11:13:40,974 INFO [StoreOpener-54ad5e2c72f82a5eca4c14122252f4d4-1 {}] regionserver.HStore(327): Store=54ad5e2c72f82a5eca4c14122252f4d4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:40,975 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,976 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,976 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,977 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,977 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,981 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=10 2024-11-07T11:13:40,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43027,1730978016820 in 498 msec 2024-11-07T11:13:40,986 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/54ad5e2c72f82a5eca4c14122252f4d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:40,987 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 54ad5e2c72f82a5eca4c14122252f4d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71327609, jitterRate=0.06286419928073883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:40,987 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:40,987 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 54ad5e2c72f82a5eca4c14122252f4d4: Running coprocessor pre-open hook at 1730978020970Writing region info on filesystem at 1730978020970Initializing all the Stores at 1730978020971 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020971Cleaning up temporary data from old regions at 1730978020977 (+6 ms)Running coprocessor post-open hooks at 1730978020987 (+10 ms)Region opened successfully at 1730978020987 2024-11-07T11:13:40,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN in 799 msec 2024-11-07T11:13:40,988 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4., pid=49, masterSystemTime=1730978020602 2024-11-07T11:13:40,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,991 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:40,991 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:40,991 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:40,992 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => f8c16ac0e25710f923881fe49bf532cf, NAME => 'test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-07T11:13:40,993 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:40,993 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:40,993 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:40,993 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:40,995 INFO [StoreOpener-f8c16ac0e25710f923881fe49bf532cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:40,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 156088cd75fa40150a09828c74f7a0a1, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,996 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:40,998 INFO [StoreOpener-f8c16ac0e25710f923881fe49bf532cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8c16ac0e25710f923881fe49bf532cf columnFamilyName family 2024-11-07T11:13:40,999 DEBUG [StoreOpener-f8c16ac0e25710f923881fe49bf532cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:40,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ad5e2c72f82a5eca4c14122252f4d4, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:40,999 INFO [StoreOpener-f8c16ac0e25710f923881fe49bf532cf-1 {}] regionserver.HStore(327): Store=f8c16ac0e25710f923881fe49bf532cf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:41,000 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,001 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,001 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,002 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,002 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=19 2024-11-07T11:13:41,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 156088cd75fa40150a09828c74f7a0a1, server=4ab3d709c6df,33951,1730978016967 in 544 msec 2024-11-07T11:13:41,004 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=156088cd75fa40150a09828c74f7a0a1, ASSIGN in 820 msec 2024-11-07T11:13:41,008 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/f8c16ac0e25710f923881fe49bf532cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:13:41,009 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened f8c16ac0e25710f923881fe49bf532cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73516577, jitterRate=0.09548236429691315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:41,009 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:41,009 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for f8c16ac0e25710f923881fe49bf532cf: Running coprocessor pre-open hook at 1730978020993Writing region info on filesystem at 1730978020993Initializing all the Stores at 1730978020995 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978020995Cleaning up temporary data from old regions at 1730978021002 (+7 ms)Running coprocessor post-open hooks at 1730978021009 (+7 ms)Region opened successfully at 1730978021009 2024-11-07T11:13:41,010 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf., pid=56, masterSystemTime=1730978020602 2024-11-07T11:13:41,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=22 2024-11-07T11:13:41,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 54ad5e2c72f82a5eca4c14122252f4d4, server=4ab3d709c6df,33951,1730978016967 in 542 msec 2024-11-07T11:13:41,013 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:41,013 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:41,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=54ad5e2c72f82a5eca4c14122252f4d4, ASSIGN in 828 msec 2024-11-07T11:13:41,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:41,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8c16ac0e25710f923881fe49bf532cf, server=4ab3d709c6df,33951,1730978016967 because future has completed 2024-11-07T11:13:41,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1aa45a9eec932f6b88e14ccca0f48aae, ASSIGN in 834 msec 2024-11-07T11:13:41,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=13 2024-11-07T11:13:41,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure f8c16ac0e25710f923881fe49bf532cf, server=4ab3d709c6df,33951,1730978016967 in 524 msec 2024-11-07T11:13:41,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f8c16ac0e25710f923881fe49bf532cf, ASSIGN in 841 msec 2024-11-07T11:13:41,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:41,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28bd8fcabc9d58f119d30748a4d3e741, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:41,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, openSeqNum=2, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:41,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1595aa61b66e39607e5d88bddc7e92d2, server=4ab3d709c6df,43401,1730978016916 because future has completed 2024-11-07T11:13:41,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-07T11:13:41,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 28bd8fcabc9d58f119d30748a4d3e741, server=4ab3d709c6df,43401,1730978016916 in 667 msec 2024-11-07T11:13:41,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=28bd8fcabc9d58f119d30748a4d3e741, ASSIGN in 853 msec 2024-11-07T11:13:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=24 2024-11-07T11:13:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 1595aa61b66e39607e5d88bddc7e92d2, server=4ab3d709c6df,43401,1730978016916 in 656 msec 2024-11-07T11:13:41,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-11-07T11:13:41,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1595aa61b66e39607e5d88bddc7e92d2, ASSIGN in 860 msec 2024-11-07T11:13:41,048 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T11:13:41,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730978021048"}]},"ts":"1730978021048"} 2024-11-07T11:13:41,052 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-07T11:13:41,053 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T11:13:41,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5300 sec 2024-11-07T11:13:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:13:41,709 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-07T11:13:41,709 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-07T11:13:41,711 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:13:41,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-07T11:13:41,736 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:13:41,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-07T11:13:41,737 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:13:41,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-07T11:13:41,768 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-07T11:13:41,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:13:41,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:13:41,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:13:41,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:13:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ebf8c75d5f559642dd7af8561a14d668 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d7939d9f92fe5210293a2217ff7af696 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f8c16ac0e25710f923881fe49bf532cf changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 78b9a3cb232752746916dce9a515cf81 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 314baf86ed26932b48f8c4cef53c1bcc changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 156088cd75fa40150a09828c74f7a0a1 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 54ad5e2c72f82a5eca4c14122252f4d4 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1aa45a9eec932f6b88e14ccca0f48aae changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dac17ebdf8c70ae5014b41449b83b0ae changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 880afe4596d2f9f36732bf50351d0096 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a5b587e52af29a25529015bea97baa17 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ba16fdeba546ef71df7c8b7d75e0fc39 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7aa9c0d5fa702e40d1908a62541f05e4 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c49878281e74ac31cf82a45639f5ef6c changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9397bd7c5b10fb3fe8f98f0911b2382b changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 89edfc8d260883f410baf2ebd451540e changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 38ac69d6e0db8e90441ae19a5f2f45e8 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 908579c522ec42c02649f6f9625ec8a4 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e6670dae2fae722a1a46cdb36b5a703a changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 356e319324497934b3703225e899ce57 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6356f249da7a2a5ae2d1836b77641b9d changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1bffef86bb7015d7d1d3993e690e3a3c changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 28bd8fcabc9d58f119d30748a4d3e741 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1595aa61b66e39607e5d88bddc7e92d2 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a32ca03dfb3b86b085738b7e4da532b9 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 80ba69ee8a10fcba947d5f7bb1fa9b5b changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:13:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-07T11:13:41,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-07T11:13:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-07T11:13:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(204): Hosts are {4ab3d709c6df=0} racks are {/default-rack=0} 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T11:13:41,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:13:41,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:13:41,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T11:13:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-07T11:13:41,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-07T11:13:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-07T11:13:41,871 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:13:41,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-07T11:13:41,903 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=2362 2024-11-07T11:13:41,906 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-07T11:13:41,907 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-07T11:13:41,907 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:41,908 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 4ab3d709c6df,43027,1730978016820: Simulated kill ***** 2024-11-07T11:13:41,908 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-07T11:13:41,910 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-07T11:13:41,913 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-07T11:13:41,913 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-07T11:13:41,913 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-07T11:13:41,914 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 299199408 }, "NonHeapMemoryUsage": { "committed": 151715840, "init": 7667712, "max": -1, "used": 149438120 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "4ab3d709c6df" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "4ab3d709c6df" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "4ab3d709c6df" } ] } 2024-11-07T11:13:41,917 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37891 {}] master.MasterRpcServices(700): 4ab3d709c6df,43027,1730978016820 reported a fatal error: ***** ABORTING region server 4ab3d709c6df,43027,1730978016820: Simulated kill ***** 2024-11-07T11:13:41,919 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4ab3d709c6df,43027,1730978016820' ***** 2024-11-07T11:13:41,920 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-07T11:13:41,920 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:13:41,920 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:13:41,920 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:13:41,920 INFO [RS:0;4ab3d709c6df:43027 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-07T11:13:41,921 INFO [RS:0;4ab3d709c6df:43027 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-07T11:13:41,921 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:13:41,921 INFO [RS:0;4ab3d709c6df:43027 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4ab3d709c6df:43027. 2024-11-07T11:13:41,921 DEBUG [RS:0;4ab3d709c6df:43027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:13:41,928 DEBUG [RS:0;4ab3d709c6df:43027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:41,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741836_1012 (size=32114) 2024-11-07T11:13:41,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741836_1012 (size=32114) 2024-11-07T11:13:41,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741836_1012 (size=32114) 2024-11-07T11:13:41,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741833_1009 (size=2232) 2024-11-07T11:13:41,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741833_1009 (size=2232) 2024-11-07T11:13:41,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741833_1009 (size=2232) 2024-11-07T11:13:41,941 DEBUG [RS:0;4ab3d709c6df:43027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:41,941 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:13:41,941 INFO [RS:0;4ab3d709c6df:43027 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43027 2024-11-07T11:13:41,947 INFO [regionserver/4ab3d709c6df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:13:42,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:13:42,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4ab3d709c6df,43027,1730978016820 2024-11-07T11:13:42,010 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:13:42,011 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4ab3d709c6df,43027,1730978016820] 2024-11-07T11:13:42,030 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4ab3d709c6df,43027,1730978016820 already deleted, retry=false 2024-11-07T11:13:42,031 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4ab3d709c6df,43027,1730978016820 on 4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:42,037 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:42,041 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for 4ab3d709c6df,43027,1730978016820 (carryingMeta=true) 4ab3d709c6df,43027,1730978016820/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@45310f07[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-07T11:13:42,042 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:42,043 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true, isMeta: true 2024-11-07T11:13:42,045 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting 2024-11-07T11:13:42,047 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4ab3d709c6df,43027,1730978016820 WAL count=1, meta=true 2024-11-07T11:13:42,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta}] 2024-11-07T11:13:42,054 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:42,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:42,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:42,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43027-0x101150063ff0001, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:42,121 INFO [RS:0;4ab3d709c6df:43027 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:13:42,122 INFO [RS:0;4ab3d709c6df:43027 {}] regionserver.HRegionServer(1031): Exiting; stopping=4ab3d709c6df,43027,1730978016820; zookeeper connection closed. 2024-11-07T11:13:42,122 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:13:42,122 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ff85ec5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ff85ec5 2024-11-07T11:13:42,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33951 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-07T11:13:42,254 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, size=31.4 K (32114bytes) 2024-11-07T11:13:42,255 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-07T11:13:42,255 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-07T11:13:42,255 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta 2024-11-07T11:13:42,260 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta after 4ms 2024-11-07T11:13:42,267 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta took 15ms 2024-11-07T11:13:42,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-07T11:13:42,283 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp 2024-11-07T11:13:42,287 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp 2024-11-07T11:13:42,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta so closing down 2024-11-07T11:13:42,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-07T11:13:42,290 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-07T11:13:42,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741864_1040 (size=31873) 2024-11-07T11:13:42,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741864_1040 (size=31873) 2024-11-07T11:13:42,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741864_1040 (size=31873) 2024-11-07T11:13:42,306 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-07T11:13:42,308 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp to hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T11:13:42,312 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-07T11:13:42,312 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, journal: Splitting hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, size=31.4 K (32114bytes) at 1730978022254Creating recovered edits writer path=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp at 1730978022287 (+33 ms)Finishing writing output for hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta so closing down at 17309780222873 split writer threads finished at 1730978022290 (+3 ms)Closed recovered edits writer path=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1730978022306 (+16 ms)Rename recovered edits hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta.temp to hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1730978022308 (+2 ms)Processed 58 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1730978022312 (+4 ms) 2024-11-07T11:13:42,313 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta 2024-11-07T11:13:42,314 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-07T11:13:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-07T11:13:42,324 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta to hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs 2024-11-07T11:13:42,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T11:13:42,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 in 267 msec 2024-11-07T11:13:42,330 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:42,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T11:13:42,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 in 282 msec 2024-11-07T11:13:42,336 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting dir is empty, no logs to split. 2024-11-07T11:13:42,336 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4ab3d709c6df,43027,1730978016820 WAL count=0, meta=true 2024-11-07T11:13:42,336 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 4ab3d709c6df,43027,1730978016820 WAL splitting is done? wals=0, meta=true 2024-11-07T11:13:42,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T11:13:42,340 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:42,341 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:42,443 INFO [regionserver/4ab3d709c6df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T11:13:42,443 INFO [regionserver/4ab3d709c6df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T11:13:42,491 DEBUG [4ab3d709c6df:37891 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T11:13:42,491 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(204): Hosts are {4ab3d709c6df=0} racks are {/default-rack=0} 2024-11-07T11:13:42,491 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:13:42,491 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:13:42,492 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:13:42,492 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:13:42,492 INFO [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:13:42,492 INFO [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:13:42,492 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:13:42,492 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:42,493 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPENING 2024-11-07T11:13:42,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:42,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:42,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:42,543 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:42,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:42,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:42,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:42,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:42,701 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T11:13:42,701 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:13:42,701 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T11:13:42,703 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4ab3d709c6df%2C43401%2C1730978016916.meta, suffix=.meta, logDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916, archiveDir=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs, maxLogs=32 2024-11-07T11:13:42,719 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916/4ab3d709c6df%2C43401%2C1730978016916.meta.1730978022706.meta, exclude list is [], retry=0 2024-11-07T11:13:42,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45515,DS-46fe5871-e6c7-4d1d-b05c-860d66f04292,DISK] 2024-11-07T11:13:42,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35325,DS-1b4b86c2-8961-491d-b7c8-90c13e356dbf,DISK] 2024-11-07T11:13:42,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44443,DS-0d44b7f8-76d0-43ee-ba6b-56a15606f3ab,DISK] 2024-11-07T11:13:42,724 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916/4ab3d709c6df%2C43401%2C1730978016916.meta.1730978022706.meta 2024-11-07T11:13:42,724 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45603:45603),(127.0.0.1/127.0.0.1:42855:42855),(127.0.0.1/127.0.0.1:39503:39503)] 2024-11-07T11:13:42,724 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:13:42,725 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T11:13:42,725 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T11:13:42,725 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T11:13:42,725 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T11:13:42,725 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:42,725 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T11:13:42,725 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T11:13:42,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T11:13:42,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:13:42,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:42,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:42,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T11:13:42,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:13:42,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:42,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:42,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T11:13:42,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:13:42,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:42,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:42,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T11:13:42,734 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:13:42,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:42,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:13:42,735 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:13:42,736 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740 2024-11-07T11:13:42,739 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740 2024-11-07T11:13:42,740 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T11:13:42,757 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T11:13:42,760 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-07T11:13:42,840 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/info/1047a54796834fc4a27564bdfd937af8 is 138, key is test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4./info:regioninfo/1730978020622/Put/seqid=0 2024-11-07T11:13:42,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741866_1042 (size=34924) 2024-11-07T11:13:42,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741866_1042 (size=34924) 2024-11-07T11:13:42,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741866_1042 (size=34924) 2024-11-07T11:13:42,853 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/info/1047a54796834fc4a27564bdfd937af8 2024-11-07T11:13:42,912 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/ns/90a72df5d825421eb5cfa1e6a935ce2e is 43, key is default/ns:d/1730978019260/Put/seqid=0 2024-11-07T11:13:42,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741867_1043 (size=5153) 2024-11-07T11:13:42,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741867_1043 (size=5153) 2024-11-07T11:13:42,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741867_1043 (size=5153) 2024-11-07T11:13:42,924 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/ns/90a72df5d825421eb5cfa1e6a935ce2e 2024-11-07T11:13:42,950 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/table/cfd8341843fa4fa5bc1689b0ec7b3471 is 40, key is test/table:state/1730978021048/Put/seqid=0 2024-11-07T11:13:42,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741868_1044 (size=5165) 2024-11-07T11:13:42,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741868_1044 (size=5165) 2024-11-07T11:13:42,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741868_1044 (size=5165) 2024-11-07T11:13:42,959 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/table/cfd8341843fa4fa5bc1689b0ec7b3471 2024-11-07T11:13:42,967 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/info/1047a54796834fc4a27564bdfd937af8 as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/info/1047a54796834fc4a27564bdfd937af8 2024-11-07T11:13:42,977 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/info/1047a54796834fc4a27564bdfd937af8, entries=260, sequenceid=60, filesize=34.1 K 2024-11-07T11:13:42,980 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/ns/90a72df5d825421eb5cfa1e6a935ce2e as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/ns/90a72df5d825421eb5cfa1e6a935ce2e 2024-11-07T11:13:42,989 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/ns/90a72df5d825421eb5cfa1e6a935ce2e, entries=2, sequenceid=60, filesize=5.0 K 2024-11-07T11:13:42,991 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/table/cfd8341843fa4fa5bc1689b0ec7b3471 as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/table/cfd8341843fa4fa5bc1689b0ec7b3471 2024-11-07T11:13:42,999 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/table/cfd8341843fa4fa5bc1689b0ec7b3471, entries=2, sequenceid=60, filesize=5.0 K 2024-11-07T11:13:43,004 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 243ms, sequenceid=60, compaction requested=false; wal=null 2024-11-07T11:13:43,005 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T11:13:43,008 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T11:13:43,009 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:13:43,009 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:13:43,010 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:13:43,011 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:13:43,013 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-07T11:13:43,014 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62517961, jitterRate=-0.06840978562831879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:13:43,014 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T11:13:43,015 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730978022726Writing region info on filesystem at 1730978022726Initializing all the Stores at 1730978022727 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978022727Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978022727Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978022727Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730978022727Obtaining lock to block concurrent updates at 1730978022760 (+33 ms)Preparing flush snapshotting stores in 1588230740 at 1730978022760Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1730978022763 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730978022763Flushing 1588230740/info: creating writer at 1730978022765 (+2 ms)Flushing 1588230740/info: appending metadata at 1730978022825 (+60 ms)Flushing 1588230740/info: closing flushed file at 1730978022829 (+4 ms)Flushing 1588230740/ns: creating writer at 1730978022896 (+67 ms)Flushing 1588230740/ns: appending metadata at 1730978022911 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1730978022911Flushing 1588230740/table: creating writer at 1730978022934 (+23 ms)Flushing 1588230740/table: appending metadata at 1730978022949 (+15 ms)Flushing 1588230740/table: closing flushed file at 1730978022949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dee065d: reopening flushed file at 1730978022966 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@435cd133: reopening flushed file at 1730978022978 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6894099d: reopening flushed file at 1730978022990 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 243ms, sequenceid=60, compaction requested=false; wal=null at 1730978023004 (+14 ms)Cleaning up temporary data from old regions at 1730978023009 (+5 ms)Running coprocessor post-open hooks at 1730978023014 (+5 ms)Region opened successfully at 1730978023014 2024-11-07T11:13:43,016 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1730978022696 2024-11-07T11:13:43,019 DEBUG [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T11:13:43,019 INFO [RS_OPEN_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T11:13:43,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,021 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:43,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,073 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T11:13:43,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 in 530 msec 2024-11-07T11:13:43,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-07T11:13:43,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 740 msec 2024-11-07T11:13:43,082 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 4ab3d709c6df,43027,1730978016820 had 9 regions 2024-11-07T11:13:43,083 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true, isMeta: false 2024-11-07T11:13:43,086 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4ab3d709c6df,43027,1730978016820 WAL count=1, meta=false 2024-11-07T11:13:43,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617}] 2024-11-07T11:13:43,089 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:43,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43401 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-07T11:13:43,267 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617, size=2.2 K (2232bytes) 2024-11-07T11:13:43,267 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 2024-11-07T11:13:43,268 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 after 1ms 2024-11-07T11:13:43,272 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 took 5ms 2024-11-07T11:13:43,275 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 38ac69d6e0db8e90441ae19a5f2f45e8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c49878281e74ac31cf82a45639f5ef6c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,278 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 89edfc8d260883f410baf2ebd451540e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,280 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ba16fdeba546ef71df7c8b7d75e0fc39: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,281 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 880afe4596d2f9f36732bf50351d0096: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,283 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7aa9c0d5fa702e40d1908a62541f05e4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,285 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a5b587e52af29a25529015bea97baa17: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9397bd7c5b10fb3fe8f98f0911b2382b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-07T11:13:43,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 so closing down 2024-11-07T11:13:43,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-07T11:13:43,287 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-07T11:13:43,288 INFO [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-11-07T11:13:43,288 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617, journal: Splitting hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617, size=2.2 K (2232bytes) at 1730978023267Finishing writing output for hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 so closing down at 1730978023287 (+20 ms)3 split writer threads finished at 1730978023287Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1730978023288 (+1 ms) 2024-11-07T11:13:43,288 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 2024-11-07T11:13:43,288 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4ab3d709c6df:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-07T11:13:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37891 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-07T11:13:43,293 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting/4ab3d709c6df%2C43027%2C1730978016820.1730978018617 to hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs 2024-11-07T11:13:43,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T11:13:43,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 in 203 msec 2024-11-07T11:13:43,297 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-07T11:13:43,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 in 211 msec 2024-11-07T11:13:43,302 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43027,1730978016820-splitting dir is empty, no logs to split. 2024-11-07T11:13:43,302 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4ab3d709c6df,43027,1730978016820 WAL count=0, meta=false 2024-11-07T11:13:43,302 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 4ab3d709c6df,43027,1730978016820 WAL splitting is done? wals=0, meta=false 2024-11-07T11:13:43,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN}] 2024-11-07T11:13:43,309 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,310 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,312 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,313 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,315 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,316 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,318 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,319 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,320 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,322 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,323 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,324 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,328 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:13:43,461 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(204): Hosts are {4ab3d709c6df=0} racks are {/default-rack=0} 2024-11-07T11:13:43,461 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:13:43,462 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:13:43,462 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:13:43,462 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:13:43,462 INFO [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:13:43,462 INFO [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:13:43,462 DEBUG [4ab3d709c6df:37891 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:13:43,462 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,463 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, regionLocation=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,464 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4ab3d709c6df:43027 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T11:13:43,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a5b587e52af29a25529015bea97baa17, regionState=OPENING, regionLocation=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T11:13:43,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 from cache 2024-11-07T11:13:43,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 4ab3d709c6df:43027 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,470 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4ab3d709c6df:43027 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T11:13:43,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43027,1730978016820, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=4ab3d709c6df:43027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 4ab3d709c6df:43027 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4ab3d709c6df/172.17.0.2:43027 2024-11-07T11:13:43,575 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:13:43,575 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4ab3d709c6df,43401,1730978016916, seqNum=-1] 2024-11-07T11:13:43,575 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:13:43,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44603, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:13:43,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN because future has completed 2024-11-07T11:13:43,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:43,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN because future has completed 2024-11-07T11:13:43,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN because future has completed 2024-11-07T11:13:43,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN because future has completed 2024-11-07T11:13:43,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN because future has completed 2024-11-07T11:13:43,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:43,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN because future has completed 2024-11-07T11:13:43,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN because future has completed 2024-11-07T11:13:43,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN because future has completed 2024-11-07T11:13:43,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:43,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:43,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916}] 2024-11-07T11:13:43,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:43,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:43,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967}] 2024-11-07T11:13:43,738 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:43,738 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => c49878281e74ac31cf82a45639f5ef6c, NAME => 'test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-07T11:13:43,739 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,739 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,739 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,739 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,741 INFO [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,742 INFO [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49878281e74ac31cf82a45639f5ef6c columnFamilyName family 2024-11-07T11:13:43,742 DEBUG [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,743 INFO [StoreOpener-c49878281e74ac31cf82a45639f5ef6c-1 {}] regionserver.HStore(327): Store=c49878281e74ac31cf82a45639f5ef6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,743 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,744 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:43,744 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => a5b587e52af29a25529015bea97baa17, NAME => 'test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-07T11:13:43,744 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,744 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,744 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,fff,1730978019494.a5b587e52af29a25529015bea97baa17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,744 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,745 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,746 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,746 INFO [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,746 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,746 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,747 INFO [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5b587e52af29a25529015bea97baa17 columnFamilyName family 2024-11-07T11:13:43,747 DEBUG [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,748 INFO [StoreOpener-a5b587e52af29a25529015bea97baa17-1 {}] regionserver.HStore(327): Store=a5b587e52af29a25529015bea97baa17/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,748 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,748 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,749 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,749 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened c49878281e74ac31cf82a45639f5ef6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60990902, jitterRate=-0.0911647379398346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,750 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:43,750 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,750 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for c49878281e74ac31cf82a45639f5ef6c: Running coprocessor pre-open hook at 1730978023739Writing region info on filesystem at 1730978023739Initializing all the Stores at 1730978023740 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023740Cleaning up temporary data from old regions at 1730978023746 (+6 ms)Running coprocessor post-open hooks at 1730978023750 (+4 ms)Region opened successfully at 1730978023750 2024-11-07T11:13:43,750 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,750 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,751 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c., pid=79, masterSystemTime=1730978023734 2024-11-07T11:13:43,752 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,753 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:43,753 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:43,753 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened a5b587e52af29a25529015bea97baa17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63537991, jitterRate=-0.05321015417575836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,754 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:43,754 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:43,754 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for a5b587e52af29a25529015bea97baa17: Running coprocessor pre-open hook at 1730978023745Writing region info on filesystem at 1730978023745Initializing all the Stores at 1730978023746 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023746Cleaning up temporary data from old regions at 1730978023750 (+4 ms)Running coprocessor post-open hooks at 1730978023754 (+4 ms)Region opened successfully at 1730978023754 2024-11-07T11:13:43,754 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => ba16fdeba546ef71df7c8b7d75e0fc39, NAME => 'test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-07T11:13:43,754 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,754 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,754 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,755 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,755 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1730978019494.a5b587e52af29a25529015bea97baa17., pid=74, masterSystemTime=1730978023739 2024-11-07T11:13:43,756 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-07T11:13:43,756 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,757 INFO [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,757 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:43,758 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:43,758 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:43,757 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,758 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 9397bd7c5b10fb3fe8f98f0911b2382b, NAME => 'test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-07T11:13:43,758 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,758 INFO [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba16fdeba546ef71df7c8b7d75e0fc39 columnFamilyName family 2024-11-07T11:13:43,758 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,758 DEBUG [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,758 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,758 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,759 INFO [StoreOpener-ba16fdeba546ef71df7c8b7d75e0fc39-1 {}] regionserver.HStore(327): Store=ba16fdeba546ef71df7c8b7d75e0fc39/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,759 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,760 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,761 INFO [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,762 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,762 INFO [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9397bd7c5b10fb3fe8f98f0911b2382b columnFamilyName family 2024-11-07T11:13:43,762 DEBUG [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,762 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,762 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,763 INFO [StoreOpener-9397bd7c5b10fb3fe8f98f0911b2382b-1 {}] regionserver.HStore(327): Store=9397bd7c5b10fb3fe8f98f0911b2382b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,763 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,764 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,765 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,766 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened ba16fdeba546ef71df7c8b7d75e0fc39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70251915, jitterRate=0.04683510959148407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,766 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,766 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:43,767 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for ba16fdeba546ef71df7c8b7d75e0fc39: Running coprocessor pre-open hook at 1730978023755Writing region info on filesystem at 1730978023755Initializing all the Stores at 1730978023756 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023756Cleaning up temporary data from old regions at 1730978023762 (+6 ms)Running coprocessor post-open hooks at 1730978023766 (+4 ms)Region opened successfully at 1730978023766 2024-11-07T11:13:43,767 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,767 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,767 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39., pid=78, masterSystemTime=1730978023734 2024-11-07T11:13:43,766 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-07T11:13:43,770 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:43,770 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,770 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,770 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:43,770 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:43,770 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:43,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 880afe4596d2f9f36732bf50351d0096, NAME => 'test,,1730978019494.880afe4596d2f9f36732bf50351d0096.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-07T11:13:43,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,771 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,,1730978019494.880afe4596d2f9f36732bf50351d0096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,771 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 9397bd7c5b10fb3fe8f98f0911b2382b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71892649, jitterRate=0.07128395140171051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,771 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:43,772 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 9397bd7c5b10fb3fe8f98f0911b2382b: Running coprocessor pre-open hook at 1730978023759Writing region info on filesystem at 1730978023759Initializing all the Stores at 1730978023760 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023760Cleaning up temporary data from old regions at 1730978023767 (+7 ms)Running coprocessor post-open hooks at 1730978023771 (+4 ms)Region opened successfully at 1730978023772 (+1 ms) 2024-11-07T11:13:43,772 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b., pid=76, masterSystemTime=1730978023739 2024-11-07T11:13:43,773 INFO [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,774 INFO [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880afe4596d2f9f36732bf50351d0096 columnFamilyName family 2024-11-07T11:13:43,774 DEBUG [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,775 INFO [StoreOpener-880afe4596d2f9f36732bf50351d0096-1 {}] regionserver.HStore(327): Store=880afe4596d2f9f36732bf50351d0096/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,775 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:43,775 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,775 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:43,775 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:43,775 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 38ac69d6e0db8e90441ae19a5f2f45e8, NAME => 'test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-07T11:13:43,776 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,776 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,776 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,776 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,776 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,777 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,777 INFO [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,778 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,778 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,778 INFO [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38ac69d6e0db8e90441ae19a5f2f45e8 columnFamilyName family 2024-11-07T11:13:43,778 DEBUG [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,779 INFO [StoreOpener-38ac69d6e0db8e90441ae19a5f2f45e8-1 {}] regionserver.HStore(327): Store=38ac69d6e0db8e90441ae19a5f2f45e8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,779 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,780 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,780 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,781 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 880afe4596d2f9f36732bf50351d0096; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59481968, jitterRate=-0.11364960670471191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,781 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:43,781 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 880afe4596d2f9f36732bf50351d0096: Running coprocessor pre-open hook at 1730978023771Writing region info on filesystem at 1730978023771Initializing all the Stores at 1730978023772 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023772Cleaning up temporary data from old regions at 1730978023778 (+6 ms)Running coprocessor post-open hooks at 1730978023781 (+3 ms)Region opened successfully at 1730978023781 2024-11-07T11:13:43,781 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,781 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,782 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,782 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1730978019494.880afe4596d2f9f36732bf50351d0096., pid=77, masterSystemTime=1730978023734 2024-11-07T11:13:43,783 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,784 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:43,784 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:43,784 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:43,784 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 89edfc8d260883f410baf2ebd451540e, NAME => 'test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-07T11:13:43,784 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 38ac69d6e0db8e90441ae19a5f2f45e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64723036, jitterRate=-0.035551607608795166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,785 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:43,785 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 38ac69d6e0db8e90441ae19a5f2f45e8: Running coprocessor pre-open hook at 1730978023776Writing region info on filesystem at 1730978023776Initializing all the Stores at 1730978023777 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023777Cleaning up temporary data from old regions at 1730978023782 (+5 ms)Running coprocessor post-open hooks at 1730978023785 (+3 ms)Region opened successfully at 1730978023785 2024-11-07T11:13:43,785 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,785 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,785 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,785 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,786 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8., pid=73, masterSystemTime=1730978023739 2024-11-07T11:13:43,787 INFO [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,788 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:43,788 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:43,788 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:43,789 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aa9c0d5fa702e40d1908a62541f05e4, NAME => 'test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-07T11:13:43,789 INFO [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89edfc8d260883f410baf2ebd451540e columnFamilyName family 2024-11-07T11:13:43,789 DEBUG [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,789 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,789 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:13:43,789 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,789 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,790 INFO [StoreOpener-89edfc8d260883f410baf2ebd451540e-1 {}] regionserver.HStore(327): Store=89edfc8d260883f410baf2ebd451540e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,790 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,791 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,791 INFO [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,792 INFO [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa9c0d5fa702e40d1908a62541f05e4 columnFamilyName family 2024-11-07T11:13:43,792 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,792 DEBUG [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:13:43,792 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,792 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,792 INFO [StoreOpener-7aa9c0d5fa702e40d1908a62541f05e4-1 {}] regionserver.HStore(327): Store=7aa9c0d5fa702e40d1908a62541f05e4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:13:43,792 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,793 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,794 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,794 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,795 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,795 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,795 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 89edfc8d260883f410baf2ebd451540e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59399316, jitterRate=-0.11488121747970581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,795 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:43,796 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 89edfc8d260883f410baf2ebd451540e: Running coprocessor pre-open hook at 1730978023785Writing region info on filesystem at 1730978023785Initializing all the Stores at 1730978023786 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023786Cleaning up temporary data from old regions at 1730978023792 (+6 ms)Running coprocessor post-open hooks at 1730978023795 (+3 ms)Region opened successfully at 1730978023795 2024-11-07T11:13:43,796 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e., pid=72, masterSystemTime=1730978023734 2024-11-07T11:13:43,797 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,798 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 7aa9c0d5fa702e40d1908a62541f05e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72543443, jitterRate=0.0809815376996994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:13:43,798 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:43,798 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 7aa9c0d5fa702e40d1908a62541f05e4: Running coprocessor pre-open hook at 1730978023789Writing region info on filesystem at 1730978023789Initializing all the Stores at 1730978023790 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730978023790Cleaning up temporary data from old regions at 1730978023795 (+5 ms)Running coprocessor post-open hooks at 1730978023798 (+3 ms)Region opened successfully at 1730978023798 2024-11-07T11:13:43,799 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4., pid=75, masterSystemTime=1730978023739 2024-11-07T11:13:43,799 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:43,799 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:43,801 DEBUG [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:43,801 INFO [RS_OPEN_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:43,825 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:43,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,826 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:43,826 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:43,827 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:43,827 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:43,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:43,827 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:43,828 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:43,831 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:43,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,842 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,843 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,843 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,844 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-07T11:13:43,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:43,846 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:43,846 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:43,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:43,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:43,865 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978053865 2024-11-07T11:13:43,865 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:43,865 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,866 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:43,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:43,867 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,868 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,897 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:43,897 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978083897 2024-11-07T11:13:43,897 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978143897 2024-11-07T11:13:43,897 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-07T11:13:43,898 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:43,898 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,898 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:43,898 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:43,898 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:43,898 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:43,898 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:43,898 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:43,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:43,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,909 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,909 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba16fdeba546ef71df7c8b7d75e0fc39, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-07T11:13:43,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:43,912 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:43,912 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:43,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:43,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:43,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:43,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:43,922 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:43,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:43,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:43,923 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:43,923 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978053923 2024-11-07T11:13:43,923 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:43,923 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,925 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:43,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:43,931 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,937 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:43,937 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978083937 2024-11-07T11:13:43,937 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978143937 2024-11-07T11:13:43,937 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-07T11:13:43,937 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:43,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,937 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:43,938 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:43,938 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:43,938 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:43,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:43,938 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:43,939 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:43,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:43,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:43,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,950 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,950 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880afe4596d2f9f36732bf50351d0096, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-07T11:13:43,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:43,952 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:43,952 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:43,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,964 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:43,964 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:43,964 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:43,964 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:43,964 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:43,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:43,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:43,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:43,965 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978053965 2024-11-07T11:13:43,965 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:43,965 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,967 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:43,968 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:43,969 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,969 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:43,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:43,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:43,977 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:43,977 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978083977 2024-11-07T11:13:43,977 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978143977 2024-11-07T11:13:43,977 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-07T11:13:43,977 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:43,977 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:43,977 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:43,978 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:43,978 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:43,978 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:43,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:43,978 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:43,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:43,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:43,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:43,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:43,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:43,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:43,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,989 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:43,989 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:43,989 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:43,989 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,989 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,989 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,989 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,989 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9397bd7c5b10fb3fe8f98f0911b2382b, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:43,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-07T11:13:43,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:43,990 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:43,990 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:43,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:43,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:43,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,001 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,001 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:44,001 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054001 2024-11-07T11:13:44,001 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,001 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,003 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,004 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,004 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,004 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,013 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,013 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084013 2024-11-07T11:13:44,013 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144013 2024-11-07T11:13:44,013 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-07T11:13:44,013 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,013 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,014 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,014 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,014 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,014 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,015 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,020 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,024 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,024 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aa9c0d5fa702e40d1908a62541f05e4, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,024 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-07T11:13:44,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,025 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,025 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,040 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,040 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:44,042 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054042 2024-11-07T11:13:44,042 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,042 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,045 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,045 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,045 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,056 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084056 2024-11-07T11:13:44,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144056 2024-11-07T11:13:44,057 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-07T11:13:44,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,057 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,057 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,057 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,058 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5b587e52af29a25529015bea97baa17, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-07T11:13:44,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,071 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,071 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:44,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054084 2024-11-07T11:13:44,085 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,085 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,088 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,088 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,093 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084093 2024-11-07T11:13:44,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144094 2024-11-07T11:13:44,094 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,094 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,094 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,094 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,094 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,101 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,101 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,101 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,101 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,102 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,102 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,102 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,102 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,102 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,102 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,102 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,102 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38ac69d6e0db8e90441ae19a5f2f45e8, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,102 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T11:13:44,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,103 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,103 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,115 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,115 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:44,116 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054116 2024-11-07T11:13:44,116 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,116 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,118 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,119 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,119 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,119 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,127 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,127 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084127 2024-11-07T11:13:44,127 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144127 2024-11-07T11:13:44,127 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-07T11:13:44,127 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,127 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,127 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,127 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,128 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,128 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89edfc8d260883f410baf2ebd451540e, server=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-07T11:13:44,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,140 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,141 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,143 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,144 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:44,155 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054155 2024-11-07T11:13:44,155 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,155 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,158 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,163 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084163 2024-11-07T11:13:44,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144163 2024-11-07T11:13:44,163 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T11:13:44,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,163 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,163 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,163 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,164 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,165 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,166 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,166 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,166 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,166 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,166 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,166 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN 2024-11-07T11:13:44,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T11:13:44,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,172 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,172 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c49878281e74ac31cf82a45639f5ef6c, ASSIGN to state=OFFLINE, location=null, table=test, region=c49878281e74ac31cf82a45639f5ef6c to restore RIT 2024-11-07T11:13:44,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054186 2024-11-07T11:13:44,186 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,186 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,187 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,189 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,198 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084198 2024-11-07T11:13:44,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144198 2024-11-07T11:13:44,198 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-07T11:13:44,198 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,198 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,198 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,199 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,199 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,199 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,199 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN 2024-11-07T11:13:44,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-07T11:13:44,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,209 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,210 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,222 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,222 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a5b587e52af29a25529015bea97baa17, ASSIGN to state=OFFLINE, location=null, table=test, region=a5b587e52af29a25529015bea97baa17 to restore RIT 2024-11-07T11:13:44,222 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054222 2024-11-07T11:13:44,222 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,222 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,225 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,225 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,225 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,231 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,231 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084231 2024-11-07T11:13:44,231 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144231 2024-11-07T11:13:44,231 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,231 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,232 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,232 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,232 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,232 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,232 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,232 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,232 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,233 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,239 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,240 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,240 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,240 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,240 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,240 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,240 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN 2024-11-07T11:13:44,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T11:13:44,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,241 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,242 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,253 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,253 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,253 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,253 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,253 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,253 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=880afe4596d2f9f36732bf50351d0096, ASSIGN to state=OFFLINE, location=null, table=test, region=880afe4596d2f9f36732bf50351d0096 to restore RIT 2024-11-07T11:13:44,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054255 2024-11-07T11:13:44,255 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,256 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,257 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,258 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,259 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,259 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,266 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,266 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084266 2024-11-07T11:13:44,266 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144266 2024-11-07T11:13:44,266 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-07T11:13:44,266 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,266 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,266 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,266 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,266 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,267 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,268 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,271 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,275 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,275 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,275 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,275 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN 2024-11-07T11:13:44,275 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T11:13:44,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,276 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,277 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,290 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,291 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,291 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,291 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,291 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,291 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7aa9c0d5fa702e40d1908a62541f05e4, ASSIGN to state=OFFLINE, location=null, table=test, region=7aa9c0d5fa702e40d1908a62541f05e4 to restore RIT 2024-11-07T11:13:44,291 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054291 2024-11-07T11:13:44,292 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,292 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,295 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,295 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,295 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,300 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,300 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084300 2024-11-07T11:13:44,300 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144300 2024-11-07T11:13:44,300 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T11:13:44,300 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,300 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,300 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,300 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,300 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,300 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,301 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,302 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN 2024-11-07T11:13:44,307 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T11:13:44,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,307 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,308 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,316 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,316 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,316 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,316 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8, ASSIGN to state=OFFLINE, location=null, table=test, region=38ac69d6e0db8e90441ae19a5f2f45e8 to restore RIT 2024-11-07T11:13:44,316 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054316 2024-11-07T11:13:44,316 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,316 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,318 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,319 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,319 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,326 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084326 2024-11-07T11:13:44,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144326 2024-11-07T11:13:44,326 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,326 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,326 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,326 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,327 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,327 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,327 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN 2024-11-07T11:13:44,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T11:13:44,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,336 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,336 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39, ASSIGN to state=OFFLINE, location=null, table=test, region=ba16fdeba546ef71df7c8b7d75e0fc39 to restore RIT 2024-11-07T11:13:44,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054347 2024-11-07T11:13:44,347 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,347 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,348 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,350 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,355 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,355 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084355 2024-11-07T11:13:44,355 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144355 2024-11-07T11:13:44,355 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-07T11:13:44,355 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,355 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,355 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,355 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,355 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,356 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,358 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,359 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,360 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,361 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,361 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,361 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN 2024-11-07T11:13:44,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T11:13:44,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,362 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,362 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,403 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,403 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=89edfc8d260883f410baf2ebd451540e, ASSIGN to state=OFFLINE, location=null, table=test, region=89edfc8d260883f410baf2ebd451540e to restore RIT 2024-11-07T11:13:44,403 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054403 2024-11-07T11:13:44,403 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,403 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,404 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,405 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,406 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,413 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084413 2024-11-07T11:13:44,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144413 2024-11-07T11:13:44,413 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,413 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,413 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,413 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,413 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,414 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,414 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,414 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,414 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,418 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,418 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,418 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,418 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,418 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,419 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,421 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,421 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN 2024-11-07T11:13:44,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T11:13:44,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,422 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,423 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b, ASSIGN to state=OFFLINE, location=null, table=test, region=9397bd7c5b10fb3fe8f98f0911b2382b to restore RIT 2024-11-07T11:13:44,443 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054443 2024-11-07T11:13:44,443 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,443 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,448 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,448 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,448 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,452 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084452 2024-11-07T11:13:44,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144452 2024-11-07T11:13:44,452 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,452 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,452 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,452 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,453 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,453 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,453 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,454 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,456 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,456 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T11:13:44,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,460 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,460 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,462 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,463 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,474 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054474 2024-11-07T11:13:44,474 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,474 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,477 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,483 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084483 2024-11-07T11:13:44,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144483 2024-11-07T11:13:44,483 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,483 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,483 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,483 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,484 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,484 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,484 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.1730978018617, worker=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T11:13:44,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,490 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,491 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054501 2024-11-07T11:13:44,501 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,501 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,503 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,504 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,510 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,510 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084510 2024-11-07T11:13:44,510 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144510 2024-11-07T11:13:44,510 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T11:13:44,510 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,511 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,511 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,511 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,511 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,511 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,511 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,511 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T11:13:44,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,516 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4ab3d709c6df,43401,1730978016916 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T11:13:44,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,519 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,519 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,533 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054533 2024-11-07T11:13:44,534 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,534 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,535 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,536 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,542 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084542 2024-11-07T11:13:44,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144542 2024-11-07T11:13:44,542 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T11:13:44,542 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,542 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,542 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,543 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,543 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,543 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T11:13:44,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:13:44,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T11:13:44,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,549 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,549 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,558 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054558 2024-11-07T11:13:44,558 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,558 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,559 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T11:13:44,559 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,560 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,560 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,561 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,566 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,566 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084566 2024-11-07T11:13:44,566 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144566 2024-11-07T11:13:44,567 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T11:13:44,567 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,567 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,567 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,567 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,567 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,568 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,571 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,571 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,571 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T11:13:44,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T11:13:44,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,574 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,575 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,575 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,575 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37891 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054590 2024-11-07T11:13:44,590 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,590 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,592 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,592 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,592 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,598 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084598 2024-11-07T11:13:44,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144598 2024-11-07T11:13:44,598 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-07T11:13:44,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,599 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,599 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,599 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,599 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T11:13:44,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 4ab3d709c6df%2C43027%2C1730978016820.meta.1730978019009.meta, worker=4ab3d709c6df,33951,1730978016967 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T11:13:44,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T11:13:44,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,605 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,605 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,616 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,616 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,616 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,617 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054617 2024-11-07T11:13:44,618 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,618 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,624 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,624 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,624 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,627 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,627 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084627 2024-11-07T11:13:44,627 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144627 2024-11-07T11:13:44,627 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-07T11:13:44,627 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,627 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T11:13:44,627 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,627 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:44,627 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:44,627 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T11:13:44,627 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T11:13:44,627 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T11:13:44,628 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:13:44,628 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:13:44,630 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T11:13:44,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:13:44,631 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-07T11:13:44,631 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true 2024-11-07T11:13:44,631 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T11:13:44,632 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T11:13:44,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=61 2024-11-07T11:13:44,632 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4ab3d709c6df,43401,1730978016916, table=hbase:meta, region=1588230740 2024-11-07T11:13:44,633 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4ab3d709c6df,43401,1730978016916, state=OPEN 2024-11-07T11:13:44,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:13:44,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:13:44,642 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730978054642 2024-11-07T11:13:44,642 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T11:13:44,642 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-07T11:13:44,643 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-07T11:13:44,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure c49878281e74ac31cf82a45639f5ef6c, server=4ab3d709c6df,33951,1730978016967:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 4ab3d709c6df,43027,1730978016820, splitWal=true, meta=true exec-time=2.6110 sec 2024-11-07T11:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=880afe4596d2f9f36732bf50351d0096, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=908579c522ec42c02649f6f9625ec8a4, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ebf8c75d5f559642dd7af8561a14d668, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6670dae2fae722a1a46cdb36b5a703a, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7939d9f92fe5210293a2217ff7af696, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5b587e52af29a25529015bea97baa17, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=356e319324497934b3703225e899ce57, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba16fdeba546ef71df7c8b7d75e0fc39, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c16ac0e25710f923881fe49bf532cf, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6356f249da7a2a5ae2d1836b77641b9d, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78b9a3cb232752746916dce9a515cf81, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=314baf86ed26932b48f8c4cef53c1bcc, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7aa9c0d5fa702e40d1908a62541f05e4, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bffef86bb7015d7d1d3993e690e3a3c, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=156088cd75fa40150a09828c74f7a0a1, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c49878281e74ac31cf82a45639f5ef6c, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=28bd8fcabc9d58f119d30748a4d3e741, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54ad5e2c72f82a5eca4c14122252f4d4, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9397bd7c5b10fb3fe8f98f0911b2382b, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1595aa61b66e39607e5d88bddc7e92d2, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1aa45a9eec932f6b88e14ccca0f48aae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89edfc8d260883f410baf2ebd451540e, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a32ca03dfb3b86b085738b7e4da532b9, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dac17ebdf8c70ae5014b41449b83b0ae, regionState=OPEN, lastHost=4ab3d709c6df,33951,1730978016967, regionLocation=4ab3d709c6df,33951,1730978016967, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=38ac69d6e0db8e90441ae19a5f2f45e8, regionState=OPENING, lastHost=4ab3d709c6df,43027,1730978016820, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80ba69ee8a10fcba947d5f7bb1fa9b5b, regionState=OPEN, lastHost=4ab3d709c6df,43401,1730978016916, regionLocation=4ab3d709c6df,43401,1730978016916, openSeqNum=2 2024-11-07T11:13:44,651 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-07T11:13:44,651 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730978084651 2024-11-07T11:13:44,652 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730978144652 2024-11-07T11:13:44,652 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-07T11:13:44,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 381), OpenFileDescriptor=612 (was 633), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 318), ProcessCount=11 (was 11), AvailableMemoryMB=2326 (was 2362) 2024-11-07T11:13:44,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T11:13:44,671 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T11:13:44,671 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:13:44,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:44,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:44,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T11:13:44,672 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T11:13:44,672 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1780335685, stopped=false 2024-11-07T11:13:44,672 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4ab3d709c6df,37891,1730978016105 2024-11-07T11:13:44,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:44,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:44,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:44,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:44,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:13:44,683 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:44,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:44,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T11:13:44,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:13:44,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:44,684 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4ab3d709c6df,43401,1730978016916' ***** 2024-11-07T11:13:44,684 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T11:13:44,684 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4ab3d709c6df,33951,1730978016967' ***** 2024-11-07T11:13:44,684 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T11:13:44,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:44,684 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:13:44,684 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:13:44,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:44,684 INFO [RS:1;4ab3d709c6df:43401 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:13:44,684 INFO [RS:2;4ab3d709c6df:33951 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:13:44,684 INFO [RS:1;4ab3d709c6df:43401 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:13:44,684 INFO [RS:2;4ab3d709c6df:33951 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:13:44,684 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:13:44,684 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:13:44,685 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for ba16fdeba546ef71df7c8b7d75e0fc39 2024-11-07T11:13:44,685 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 1595aa61b66e39607e5d88bddc7e92d2 2024-11-07T11:13:44,685 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:13:44,685 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 314baf86ed26932b48f8c4cef53c1bcc 2024-11-07T11:13:44,685 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for d7939d9f92fe5210293a2217ff7af696 2024-11-07T11:13:44,685 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for c49878281e74ac31cf82a45639f5ef6c 2024-11-07T11:13:44,685 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 880afe4596d2f9f36732bf50351d0096 2024-11-07T11:13:44,685 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 78b9a3cb232752746916dce9a515cf81 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 156088cd75fa40150a09828c74f7a0a1 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for ebf8c75d5f559642dd7af8561a14d668 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for dac17ebdf8c70ae5014b41449b83b0ae 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 54ad5e2c72f82a5eca4c14122252f4d4 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 1aa45a9eec932f6b88e14ccca0f48aae 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 89edfc8d260883f410baf2ebd451540e 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(959): stopping server 4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4ab3d709c6df:33951. 2024-11-07T11:13:44,686 DEBUG [RS:2;4ab3d709c6df:33951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:13:44,686 DEBUG [RS:2;4ab3d709c6df:33951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:44,686 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba16fdeba546ef71df7c8b7d75e0fc39, disabling compactions & flushes 2024-11-07T11:13:44,686 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:44,686 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. after waiting 0 ms 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:44,687 DEBUG [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1351): Waiting on 156088cd75fa40150a09828c74f7a0a1, 1aa45a9eec932f6b88e14ccca0f48aae, 314baf86ed26932b48f8c4cef53c1bcc, 54ad5e2c72f82a5eca4c14122252f4d4, 78b9a3cb232752746916dce9a515cf81, 880afe4596d2f9f36732bf50351d0096, 89edfc8d260883f410baf2ebd451540e, ba16fdeba546ef71df7c8b7d75e0fc39, c49878281e74ac31cf82a45639f5ef6c, d7939d9f92fe5210293a2217ff7af696, dac17ebdf8c70ae5014b41449b83b0ae, ebf8c75d5f559642dd7af8561a14d668, f8c16ac0e25710f923881fe49bf532cf 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 80ba69ee8a10fcba947d5f7bb1fa9b5b 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 6356f249da7a2a5ae2d1836b77641b9d 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 9397bd7c5b10fb3fe8f98f0911b2382b 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 38ac69d6e0db8e90441ae19a5f2f45e8 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1595aa61b66e39607e5d88bddc7e92d2, disabling compactions & flushes 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 28bd8fcabc9d58f119d30748a4d3e741 2024-11-07T11:13:44,687 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 7aa9c0d5fa702e40d1908a62541f05e4 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for a32ca03dfb3b86b085738b7e4da532b9 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. after waiting 0 ms 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for a5b587e52af29a25529015bea97baa17 2024-11-07T11:13:44,687 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 1bffef86bb7015d7d1d3993e690e3a3c 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 356e319324497934b3703225e899ce57 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 908579c522ec42c02649f6f9625ec8a4 2024-11-07T11:13:44,687 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(959): stopping server 4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4ab3d709c6df:43401. 2024-11-07T11:13:44,688 DEBUG [RS:1;4ab3d709c6df:43401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:13:44,688 DEBUG [RS:1;4ab3d709c6df:43401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T11:13:44,688 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-07T11:13:44,689 DEBUG [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1595aa61b66e39607e5d88bddc7e92d2, 1bffef86bb7015d7d1d3993e690e3a3c, 28bd8fcabc9d58f119d30748a4d3e741, 356e319324497934b3703225e899ce57, 38ac69d6e0db8e90441ae19a5f2f45e8, 6356f249da7a2a5ae2d1836b77641b9d, 7aa9c0d5fa702e40d1908a62541f05e4, 80ba69ee8a10fcba947d5f7bb1fa9b5b, 908579c522ec42c02649f6f9625ec8a4, 9397bd7c5b10fb3fe8f98f0911b2382b, a32ca03dfb3b86b085738b7e4da532b9, a5b587e52af29a25529015bea97baa17, e6670dae2fae722a1a46cdb36b5a703a 2024-11-07T11:13:44,689 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T11:13:44,689 INFO [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T11:13:44,689 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T11:13:44,689 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T11:13:44,689 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T11:13:44,689 INFO [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-07T11:13:44,693 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1595aa61b66e39607e5d88bddc7e92d2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,694 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ba16fdeba546ef71df7c8b7d75e0fc39/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,696 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:44,696 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1595aa61b66e39607e5d88bddc7e92d2: Waiting for close lock at 1730978024687Running coprocessor pre-close hooks at 1730978024687Disabling compacts and flushes for region at 1730978024687Disabling writes for close at 1730978024687Writing region close event to WAL at 1730978024688 (+1 ms)Running coprocessor post-close hooks at 1730978024694 (+6 ms)Closed at 1730978024696 (+2 ms) 2024-11-07T11:13:44,696 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:44,696 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba16fdeba546ef71df7c8b7d75e0fc39: Waiting for close lock at 1730978024685Running coprocessor pre-close hooks at 1730978024686 (+1 ms)Disabling compacts and flushes for region at 1730978024686Disabling writes for close at 1730978024687 (+1 ms)Writing region close event to WAL at 1730978024687Running coprocessor post-close hooks at 1730978024696 (+9 ms)Closed at 1730978024696 2024-11-07T11:13:44,696 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1730978019494.ba16fdeba546ef71df7c8b7d75e0fc39. 2024-11-07T11:13:44,696 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1730978019494.1595aa61b66e39607e5d88bddc7e92d2. 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 314baf86ed26932b48f8c4cef53c1bcc, disabling compactions & flushes 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80ba69ee8a10fcba947d5f7bb1fa9b5b, disabling compactions & flushes 2024-11-07T11:13:44,697 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:44,697 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. after waiting 0 ms 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. after waiting 0 ms 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:44,697 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:44,703 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/314baf86ed26932b48f8c4cef53c1bcc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,703 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/80ba69ee8a10fcba947d5f7bb1fa9b5b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,704 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:44,704 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 314baf86ed26932b48f8c4cef53c1bcc: Waiting for close lock at 1730978024697Running coprocessor pre-close hooks at 1730978024697Disabling compacts and flushes for region at 1730978024697Disabling writes for close at 1730978024697Writing region close event to WAL at 1730978024697Running coprocessor post-close hooks at 1730978024703 (+6 ms)Closed at 1730978024704 (+1 ms) 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80ba69ee8a10fcba947d5f7bb1fa9b5b: Waiting for close lock at 1730978024697Running coprocessor pre-close hooks at 1730978024697Disabling compacts and flushes for region at 1730978024697Disabling writes for close at 1730978024697Writing region close event to WAL at 1730978024698 (+1 ms)Running coprocessor post-close hooks at 1730978024704 (+6 ms)Closed at 1730978024704 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1730978019494.80ba69ee8a10fcba947d5f7bb1fa9b5b. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1730978019494.314baf86ed26932b48f8c4cef53c1bcc. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6356f249da7a2a5ae2d1836b77641b9d, disabling compactions & flushes 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d7939d9f92fe5210293a2217ff7af696, disabling compactions & flushes 2024-11-07T11:13:44,704 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:44,704 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. after waiting 0 ms 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. after waiting 0 ms 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:44,704 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:44,706 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/info/7d375aa65d6a4e5188ebb0a583420413 is 138, key is test,fff,1730978019494.a5b587e52af29a25529015bea97baa17./info:regioninfo/1730978023466/Put/seqid=0 2024-11-07T11:13:44,710 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/6356f249da7a2a5ae2d1836b77641b9d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,711 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/d7939d9f92fe5210293a2217ff7af696/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,711 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:44,711 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:44,711 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d7939d9f92fe5210293a2217ff7af696: Waiting for close lock at 1730978024704Running coprocessor pre-close hooks at 1730978024704Disabling compacts and flushes for region at 1730978024704Disabling writes for close at 1730978024704Writing region close event to WAL at 1730978024705 (+1 ms)Running coprocessor post-close hooks at 1730978024711 (+6 ms)Closed at 1730978024711 2024-11-07T11:13:44,711 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6356f249da7a2a5ae2d1836b77641b9d: Waiting for close lock at 1730978024704Running coprocessor pre-close hooks at 1730978024704Disabling compacts and flushes for region at 1730978024704Disabling writes for close at 1730978024704Writing region close event to WAL at 1730978024705 (+1 ms)Running coprocessor post-close hooks at 1730978024711 (+6 ms)Closed at 1730978024711 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1730978019494.d7939d9f92fe5210293a2217ff7af696. 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1730978019494.6356f249da7a2a5ae2d1836b77641b9d. 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c49878281e74ac31cf82a45639f5ef6c, disabling compactions & flushes 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9397bd7c5b10fb3fe8f98f0911b2382b, disabling compactions & flushes 2024-11-07T11:13:44,712 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:44,712 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. after waiting 0 ms 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. after waiting 0 ms 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:44,712 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:44,722 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/9397bd7c5b10fb3fe8f98f0911b2382b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,723 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:44,723 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9397bd7c5b10fb3fe8f98f0911b2382b: Waiting for close lock at 1730978024712Running coprocessor pre-close hooks at 1730978024712Disabling compacts and flushes for region at 1730978024712Disabling writes for close at 1730978024712Writing region close event to WAL at 1730978024712Running coprocessor post-close hooks at 1730978024723 (+11 ms)Closed at 1730978024723 2024-11-07T11:13:44,723 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1730978019494.9397bd7c5b10fb3fe8f98f0911b2382b. 2024-11-07T11:13:44,724 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38ac69d6e0db8e90441ae19a5f2f45e8, disabling compactions & flushes 2024-11-07T11:13:44,724 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:44,724 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:44,724 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. after waiting 0 ms 2024-11-07T11:13:44,724 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:44,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741869_1045 (size=8177) 2024-11-07T11:13:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741869_1045 (size=8177) 2024-11-07T11:13:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741869_1045 (size=8177) 2024-11-07T11:13:44,728 INFO [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/info/7d375aa65d6a4e5188ebb0a583420413 2024-11-07T11:13:44,729 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/c49878281e74ac31cf82a45639f5ef6c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,730 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:44,730 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c49878281e74ac31cf82a45639f5ef6c: Waiting for close lock at 1730978024712Running coprocessor pre-close hooks at 1730978024712Disabling compacts and flushes for region at 1730978024712Disabling writes for close at 1730978024712Writing region close event to WAL at 1730978024716 (+4 ms)Running coprocessor post-close hooks at 1730978024730 (+14 ms)Closed at 1730978024730 2024-11-07T11:13:44,731 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1730978019494.c49878281e74ac31cf82a45639f5ef6c. 2024-11-07T11:13:44,731 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 880afe4596d2f9f36732bf50351d0096, disabling compactions & flushes 2024-11-07T11:13:44,731 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:44,731 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:44,731 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1730978019494.880afe4596d2f9f36732bf50351d0096. after waiting 0 ms 2024-11-07T11:13:44,731 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:44,734 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/38ac69d6e0db8e90441ae19a5f2f45e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,735 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:44,735 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38ac69d6e0db8e90441ae19a5f2f45e8: Waiting for close lock at 1730978024724Running coprocessor pre-close hooks at 1730978024724Disabling compacts and flushes for region at 1730978024724Disabling writes for close at 1730978024724Writing region close event to WAL at 1730978024728 (+4 ms)Running coprocessor post-close hooks at 1730978024735 (+7 ms)Closed at 1730978024735 2024-11-07T11:13:44,735 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1730978019494.38ac69d6e0db8e90441ae19a5f2f45e8. 2024-11-07T11:13:44,735 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e6670dae2fae722a1a46cdb36b5a703a, disabling compactions & flushes 2024-11-07T11:13:44,735 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:44,736 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:44,736 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. after waiting 0 ms 2024-11-07T11:13:44,736 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:44,736 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/880afe4596d2f9f36732bf50351d0096/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,737 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:44,738 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 880afe4596d2f9f36732bf50351d0096: Waiting for close lock at 1730978024731Running coprocessor pre-close hooks at 1730978024731Disabling compacts and flushes for region at 1730978024731Disabling writes for close at 1730978024731Writing region close event to WAL at 1730978024732 (+1 ms)Running coprocessor post-close hooks at 1730978024737 (+5 ms)Closed at 1730978024737 2024-11-07T11:13:44,738 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1730978019494.880afe4596d2f9f36732bf50351d0096. 2024-11-07T11:13:44,738 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 78b9a3cb232752746916dce9a515cf81, disabling compactions & flushes 2024-11-07T11:13:44,738 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:44,738 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:44,738 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. after waiting 0 ms 2024-11-07T11:13:44,738 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:44,740 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/.tmp/info/7d375aa65d6a4e5188ebb0a583420413 as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/info/7d375aa65d6a4e5188ebb0a583420413 2024-11-07T11:13:44,741 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/e6670dae2fae722a1a46cdb36b5a703a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,742 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:44,742 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e6670dae2fae722a1a46cdb36b5a703a: Waiting for close lock at 1730978024735Running coprocessor pre-close hooks at 1730978024735Disabling compacts and flushes for region at 1730978024735Disabling writes for close at 1730978024736 (+1 ms)Writing region close event to WAL at 1730978024736Running coprocessor post-close hooks at 1730978024742 (+6 ms)Closed at 1730978024742 2024-11-07T11:13:44,742 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1730978019494.e6670dae2fae722a1a46cdb36b5a703a. 2024-11-07T11:13:44,742 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 28bd8fcabc9d58f119d30748a4d3e741, disabling compactions & flushes 2024-11-07T11:13:44,743 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:44,743 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:44,743 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. after waiting 0 ms 2024-11-07T11:13:44,743 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:44,745 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/78b9a3cb232752746916dce9a515cf81/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,746 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:44,746 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 78b9a3cb232752746916dce9a515cf81: Waiting for close lock at 1730978024738Running coprocessor pre-close hooks at 1730978024738Disabling compacts and flushes for region at 1730978024738Disabling writes for close at 1730978024738Writing region close event to WAL at 1730978024739 (+1 ms)Running coprocessor post-close hooks at 1730978024746 (+7 ms)Closed at 1730978024746 2024-11-07T11:13:44,746 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1730978019494.78b9a3cb232752746916dce9a515cf81. 2024-11-07T11:13:44,747 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 156088cd75fa40150a09828c74f7a0a1, disabling compactions & flushes 2024-11-07T11:13:44,747 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:44,747 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:44,747 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. after waiting 0 ms 2024-11-07T11:13:44,747 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:44,748 INFO [regionserver/4ab3d709c6df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:13:44,749 INFO [regionserver/4ab3d709c6df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:13:44,750 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/28bd8fcabc9d58f119d30748a4d3e741/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,751 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:44,751 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 28bd8fcabc9d58f119d30748a4d3e741: Waiting for close lock at 1730978024742Running coprocessor pre-close hooks at 1730978024742Disabling compacts and flushes for region at 1730978024742Disabling writes for close at 1730978024743 (+1 ms)Writing region close event to WAL at 1730978024743Running coprocessor post-close hooks at 1730978024750 (+7 ms)Closed at 1730978024751 (+1 ms) 2024-11-07T11:13:44,751 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1730978019494.28bd8fcabc9d58f119d30748a4d3e741. 2024-11-07T11:13:44,751 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7aa9c0d5fa702e40d1908a62541f05e4, disabling compactions & flushes 2024-11-07T11:13:44,751 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:44,751 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:44,751 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. after waiting 0 ms 2024-11-07T11:13:44,751 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:44,752 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/156088cd75fa40150a09828c74f7a0a1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,753 INFO [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/info/7d375aa65d6a4e5188ebb0a583420413, entries=24, sequenceid=71, filesize=8.0 K 2024-11-07T11:13:44,753 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:44,753 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 156088cd75fa40150a09828c74f7a0a1: Waiting for close lock at 1730978024747Running coprocessor pre-close hooks at 1730978024747Disabling compacts and flushes for region at 1730978024747Disabling writes for close at 1730978024747Writing region close event to WAL at 1730978024747Running coprocessor post-close hooks at 1730978024753 (+6 ms)Closed at 1730978024753 2024-11-07T11:13:44,753 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1730978019494.156088cd75fa40150a09828c74f7a0a1. 2024-11-07T11:13:44,754 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ebf8c75d5f559642dd7af8561a14d668, disabling compactions & flushes 2024-11-07T11:13:44,754 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:44,754 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:44,754 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. after waiting 0 ms 2024-11-07T11:13:44,754 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:44,755 INFO [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=71, compaction requested=false 2024-11-07T11:13:44,757 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/7aa9c0d5fa702e40d1908a62541f05e4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,759 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:44,759 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7aa9c0d5fa702e40d1908a62541f05e4: Waiting for close lock at 1730978024751Running coprocessor pre-close hooks at 1730978024751Disabling compacts and flushes for region at 1730978024751Disabling writes for close at 1730978024751Writing region close event to WAL at 1730978024752 (+1 ms)Running coprocessor post-close hooks at 1730978024759 (+7 ms)Closed at 1730978024759 2024-11-07T11:13:44,759 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1730978019494.7aa9c0d5fa702e40d1908a62541f05e4. 2024-11-07T11:13:44,759 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a32ca03dfb3b86b085738b7e4da532b9, disabling compactions & flushes 2024-11-07T11:13:44,759 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:44,760 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:44,760 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/ebf8c75d5f559642dd7af8561a14d668/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,760 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. after waiting 0 ms 2024-11-07T11:13:44,760 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:44,760 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:44,760 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ebf8c75d5f559642dd7af8561a14d668: Waiting for close lock at 1730978024754Running coprocessor pre-close hooks at 1730978024754Disabling compacts and flushes for region at 1730978024754Disabling writes for close at 1730978024754Writing region close event to WAL at 1730978024754Running coprocessor post-close hooks at 1730978024760 (+6 ms)Closed at 1730978024760 2024-11-07T11:13:44,761 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1730978019494.ebf8c75d5f559642dd7af8561a14d668. 2024-11-07T11:13:44,761 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dac17ebdf8c70ae5014b41449b83b0ae, disabling compactions & flushes 2024-11-07T11:13:44,761 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:44,761 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:44,761 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. after waiting 0 ms 2024-11-07T11:13:44,761 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:44,765 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-07T11:13:44,765 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/dac17ebdf8c70ae5014b41449b83b0ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,765 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a32ca03dfb3b86b085738b7e4da532b9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,766 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dac17ebdf8c70ae5014b41449b83b0ae: Waiting for close lock at 1730978024761Running coprocessor pre-close hooks at 1730978024761Disabling compacts and flushes for region at 1730978024761Disabling writes for close at 1730978024761Writing region close event to WAL at 1730978024761Running coprocessor post-close hooks at 1730978024766 (+5 ms)Closed at 1730978024766 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1730978019494.dac17ebdf8c70ae5014b41449b83b0ae. 2024-11-07T11:13:44,766 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:44,766 INFO [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a32ca03dfb3b86b085738b7e4da532b9: Waiting for close lock at 1730978024759Running coprocessor pre-close hooks at 1730978024759Disabling compacts and flushes for region at 1730978024759Disabling writes for close at 1730978024760 (+1 ms)Writing region close event to WAL at 1730978024760Running coprocessor post-close hooks at 1730978024766 (+6 ms)Closed at 1730978024766 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730978024689Running coprocessor pre-close hooks at 1730978024689Disabling compacts and flushes for region at 1730978024689Disabling writes for close at 1730978024689Obtaining lock to block concurrent updates at 1730978024689Preparing flush snapshotting stores in 1588230740 at 1730978024689Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1730978024698 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730978024700 (+2 ms)Flushing 1588230740/info: creating writer at 1730978024700Flushing 1588230740/info: appending metadata at 1730978024705 (+5 ms)Flushing 1588230740/info: closing flushed file at 1730978024705Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@321f24cf: reopening flushed file at 1730978024739 (+34 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=71, compaction requested=false at 1730978024755 (+16 ms)Writing region close event to WAL at 1730978024758 (+3 ms)Running coprocessor post-close hooks at 1730978024766 (+8 ms)Closed at 1730978024766 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 54ad5e2c72f82a5eca4c14122252f4d4, disabling compactions & flushes 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1730978019494.a32ca03dfb3b86b085738b7e4da532b9. 2024-11-07T11:13:44,766 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_META-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. after waiting 0 ms 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5b587e52af29a25529015bea97baa17, disabling compactions & flushes 2024-11-07T11:13:44,766 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:44,766 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:44,767 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:44,767 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. after waiting 0 ms 2024-11-07T11:13:44,767 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:44,771 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/a5b587e52af29a25529015bea97baa17/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,772 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/54ad5e2c72f82a5eca4c14122252f4d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5b587e52af29a25529015bea97baa17: Waiting for close lock at 1730978024766Running coprocessor pre-close hooks at 1730978024766Disabling compacts and flushes for region at 1730978024766Disabling writes for close at 1730978024767 (+1 ms)Writing region close event to WAL at 1730978024767Running coprocessor post-close hooks at 1730978024772 (+5 ms)Closed at 1730978024772 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1730978019494.a5b587e52af29a25529015bea97baa17. 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1bffef86bb7015d7d1d3993e690e3a3c, disabling compactions & flushes 2024-11-07T11:13:44,772 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. after waiting 0 ms 2024-11-07T11:13:44,772 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:44,773 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:44,773 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 54ad5e2c72f82a5eca4c14122252f4d4: Waiting for close lock at 1730978024766Running coprocessor pre-close hooks at 1730978024766Disabling compacts and flushes for region at 1730978024766Disabling writes for close at 1730978024766Writing region close event to WAL at 1730978024767 (+1 ms)Running coprocessor post-close hooks at 1730978024773 (+6 ms)Closed at 1730978024773 2024-11-07T11:13:44,773 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1730978019494.54ad5e2c72f82a5eca4c14122252f4d4. 2024-11-07T11:13:44,773 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f8c16ac0e25710f923881fe49bf532cf, disabling compactions & flushes 2024-11-07T11:13:44,773 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:44,773 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:44,773 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. after waiting 0 ms 2024-11-07T11:13:44,773 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:44,777 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1bffef86bb7015d7d1d3993e690e3a3c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,777 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/f8c16ac0e25710f923881fe49bf532cf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,778 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:44,778 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1bffef86bb7015d7d1d3993e690e3a3c: Waiting for close lock at 1730978024772Running coprocessor pre-close hooks at 1730978024772Disabling compacts and flushes for region at 1730978024772Disabling writes for close at 1730978024772Writing region close event to WAL at 1730978024773 (+1 ms)Running coprocessor post-close hooks at 1730978024777 (+4 ms)Closed at 1730978024778 (+1 ms) 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f8c16ac0e25710f923881fe49bf532cf: Waiting for close lock at 1730978024773Running coprocessor pre-close hooks at 1730978024773Disabling compacts and flushes for region at 1730978024773Disabling writes for close at 1730978024773Writing region close event to WAL at 1730978024773Running coprocessor post-close hooks at 1730978024778 (+5 ms)Closed at 1730978024778 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1730978019494.1bffef86bb7015d7d1d3993e690e3a3c. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1730978019494.f8c16ac0e25710f923881fe49bf532cf. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 356e319324497934b3703225e899ce57, disabling compactions & flushes 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1aa45a9eec932f6b88e14ccca0f48aae, disabling compactions & flushes 2024-11-07T11:13:44,778 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:44,778 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. after waiting 0 ms 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1730978019494.356e319324497934b3703225e899ce57. after waiting 0 ms 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:44,782 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/356e319324497934b3703225e899ce57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,782 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/1aa45a9eec932f6b88e14ccca0f48aae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,783 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:44,783 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1aa45a9eec932f6b88e14ccca0f48aae: Waiting for close lock at 1730978024778Running coprocessor pre-close hooks at 1730978024778Disabling compacts and flushes for region at 1730978024778Disabling writes for close at 1730978024778Writing region close event to WAL at 1730978024779 (+1 ms)Running coprocessor post-close hooks at 1730978024783 (+4 ms)Closed at 1730978024783 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 356e319324497934b3703225e899ce57: Waiting for close lock at 1730978024778Running coprocessor pre-close hooks at 1730978024778Disabling compacts and flushes for region at 1730978024778Disabling writes for close at 1730978024778Writing region close event to WAL at 1730978024779 (+1 ms)Running coprocessor post-close hooks at 1730978024783 (+4 ms)Closed at 1730978024783 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1730978019494.356e319324497934b3703225e899ce57. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1730978019494.1aa45a9eec932f6b88e14ccca0f48aae. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 908579c522ec42c02649f6f9625ec8a4, disabling compactions & flushes 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 89edfc8d260883f410baf2ebd451540e, disabling compactions & flushes 2024-11-07T11:13:44,783 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:44,783 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. after waiting 0 ms 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. after waiting 0 ms 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:44,783 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:44,787 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/908579c522ec42c02649f6f9625ec8a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,787 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/data/default/test/89edfc8d260883f410baf2ebd451540e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T11:13:44,788 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:44,788 INFO [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:44,788 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 908579c522ec42c02649f6f9625ec8a4: Waiting for close lock at 1730978024783Running coprocessor pre-close hooks at 1730978024783Disabling compacts and flushes for region at 1730978024783Disabling writes for close at 1730978024783Writing region close event to WAL at 1730978024784 (+1 ms)Running coprocessor post-close hooks at 1730978024788 (+4 ms)Closed at 1730978024788 2024-11-07T11:13:44,788 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 89edfc8d260883f410baf2ebd451540e: Waiting for close lock at 1730978024783Running coprocessor pre-close hooks at 1730978024783Disabling compacts and flushes for region at 1730978024783Disabling writes for close at 1730978024783Writing region close event to WAL at 1730978024784 (+1 ms)Running coprocessor post-close hooks at 1730978024788 (+4 ms)Closed at 1730978024788 2024-11-07T11:13:44,788 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1730978019494.908579c522ec42c02649f6f9625ec8a4. 2024-11-07T11:13:44,788 DEBUG [RS_CLOSE_REGION-regionserver/4ab3d709c6df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1730978019494.89edfc8d260883f410baf2ebd451540e. 2024-11-07T11:13:44,887 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(976): stopping server 4ab3d709c6df,33951,1730978016967; all regions closed. 2024-11-07T11:13:44,889 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(976): stopping server 4ab3d709c6df,43401,1730978016916; all regions closed. 2024-11-07T11:13:44,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741834_1010 (size=7064) 2024-11-07T11:13:44,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741834_1010 (size=7064) 2024-11-07T11:13:44,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741834_1010 (size=7064) 2024-11-07T11:13:44,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741865_1041 (size=4192) 2024-11-07T11:13:44,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741865_1041 (size=4192) 2024-11-07T11:13:44,893 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/WALs/4ab3d709c6df,43401,1730978016916/4ab3d709c6df%2C43401%2C1730978016916.meta.1730978022706.meta not finished, retry = 0 2024-11-07T11:13:44,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741865_1041 (size=4192) 2024-11-07T11:13:44,898 DEBUG [RS:2;4ab3d709c6df:33951 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs 2024-11-07T11:13:44,898 INFO [RS:2;4ab3d709c6df:33951 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4ab3d709c6df%2C33951%2C1730978016967:(num 1730978018617) 2024-11-07T11:13:44,898 DEBUG [RS:2;4ab3d709c6df:33951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:44,898 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:13:44,898 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:13:44,898 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.ChoreService(370): Chore service for: regionserver/4ab3d709c6df:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T11:13:44,899 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:13:44,899 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:13:44,899 INFO [regionserver/4ab3d709c6df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:13:44,899 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:13:44,899 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:13:44,899 INFO [RS:2;4ab3d709c6df:33951 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33951 2024-11-07T11:13:44,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:13:44,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4ab3d709c6df,33951,1730978016967 2024-11-07T11:13:44,911 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:13:44,912 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4ab3d709c6df,33951,1730978016967] 2024-11-07T11:13:44,932 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4ab3d709c6df,33951,1730978016967 already deleted, retry=false 2024-11-07T11:13:44,932 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4ab3d709c6df,33951,1730978016967 expired; onlineServers=1 2024-11-07T11:13:44,998 DEBUG [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs 2024-11-07T11:13:44,998 INFO [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4ab3d709c6df%2C43401%2C1730978016916.meta:.meta(num 1730978022706) 2024-11-07T11:13:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741835_1011 (size=7074) 2024-11-07T11:13:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741835_1011 (size=7074) 2024-11-07T11:13:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741835_1011 (size=7074) 2024-11-07T11:13:45,005 DEBUG [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/oldWALs 2024-11-07T11:13:45,005 INFO [RS:1;4ab3d709c6df:43401 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4ab3d709c6df%2C43401%2C1730978016916:(num 1730978018617) 2024-11-07T11:13:45,005 DEBUG [RS:1;4ab3d709c6df:43401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:13:45,005 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:13:45,005 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:13:45,005 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.ChoreService(370): Chore service for: regionserver/4ab3d709c6df:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T11:13:45,005 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:13:45,005 INFO [regionserver/4ab3d709c6df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:13:45,005 INFO [RS:1;4ab3d709c6df:43401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43401 2024-11-07T11:13:45,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:13:45,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4ab3d709c6df,43401,1730978016916 2024-11-07T11:13:45,016 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:13:45,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:45,021 INFO [RS:2;4ab3d709c6df:33951 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:13:45,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x101150063ff0003, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:45,021 INFO [RS:2;4ab3d709c6df:33951 {}] regionserver.HRegionServer(1031): Exiting; stopping=4ab3d709c6df,33951,1730978016967; zookeeper connection closed. 2024-11-07T11:13:45,022 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79d9d510 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79d9d510 2024-11-07T11:13:45,026 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4ab3d709c6df,43401,1730978016916] 2024-11-07T11:13:45,037 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4ab3d709c6df,43401,1730978016916 already deleted, retry=false 2024-11-07T11:13:45,037 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4ab3d709c6df,43401,1730978016916 expired; onlineServers=0 2024-11-07T11:13:45,037 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4ab3d709c6df,37891,1730978016105' ***** 2024-11-07T11:13:45,037 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T11:13:45,037 INFO [M:0;4ab3d709c6df:37891 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:13:45,037 INFO [M:0;4ab3d709c6df:37891 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:13:45,037 DEBUG [M:0;4ab3d709c6df:37891 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T11:13:45,038 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T11:13:45,038 DEBUG [M:0;4ab3d709c6df:37891 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T11:13:45,038 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster-HFileCleaner.small.0-1730978018155 {}] cleaner.HFileCleaner(306): Exit Thread[master/4ab3d709c6df:0:becomeActiveMaster-HFileCleaner.small.0-1730978018155,5,FailOnTimeoutGroup] 2024-11-07T11:13:45,038 DEBUG [master/4ab3d709c6df:0:becomeActiveMaster-HFileCleaner.large.0-1730978018154 {}] cleaner.HFileCleaner(306): Exit Thread[master/4ab3d709c6df:0:becomeActiveMaster-HFileCleaner.large.0-1730978018154,5,FailOnTimeoutGroup] 2024-11-07T11:13:45,038 INFO [M:0;4ab3d709c6df:37891 {}] hbase.ChoreService(370): Chore service for: master/4ab3d709c6df:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T11:13:45,038 INFO [M:0;4ab3d709c6df:37891 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:13:45,038 DEBUG [M:0;4ab3d709c6df:37891 {}] master.HMaster(1795): Stopping service threads 2024-11-07T11:13:45,038 INFO [M:0;4ab3d709c6df:37891 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T11:13:45,038 INFO [M:0;4ab3d709c6df:37891 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:13:45,038 INFO [M:0;4ab3d709c6df:37891 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:13:45,039 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T11:13:45,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T11:13:45,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:13:45,048 DEBUG [M:0;4ab3d709c6df:37891 {}] zookeeper.ZKUtil(347): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T11:13:45,048 WARN [M:0;4ab3d709c6df:37891 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T11:13:45,048 INFO [M:0;4ab3d709c6df:37891 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/.lastflushedseqids 2024-11-07T11:13:45,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741870_1046 (size=1) 2024-11-07T11:13:45,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741870_1046 (size=1) 2024-11-07T11:13:45,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741870_1046 (size=1) 2024-11-07T11:13:45,059 INFO [M:0;4ab3d709c6df:37891 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:13:45,059 INFO [M:0;4ab3d709c6df:37891 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T11:13:45,059 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T11:13:45,059 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:45,059 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:45,059 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T11:13:45,059 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:45,060 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.05 KB heapSize=303.89 KB 2024-11-07T11:13:45,076 DEBUG [M:0;4ab3d709c6df:37891 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b66b7b61ceb740209792c4bf904fef81 is 82, key is hbase:meta,,1/info:regioninfo/1730978023020/Put/seqid=0 2024-11-07T11:13:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741871_1047 (size=6071) 2024-11-07T11:13:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741871_1047 (size=6071) 2024-11-07T11:13:45,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741871_1047 (size=6071) 2024-11-07T11:13:45,085 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b66b7b61ceb740209792c4bf904fef81 2024-11-07T11:13:45,112 DEBUG [M:0;4ab3d709c6df:37891 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/90aa978450154cc08e742731ce559969 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1730978024644/Put/seqid=0 2024-11-07T11:13:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741872_1048 (size=12797) 2024-11-07T11:13:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741872_1048 (size=12797) 2024-11-07T11:13:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741872_1048 (size=12797) 2024-11-07T11:13:45,123 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.77 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/90aa978450154cc08e742731ce559969 2024-11-07T11:13:45,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:45,127 INFO [RS:1;4ab3d709c6df:43401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:13:45,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43401-0x101150063ff0002, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:45,127 INFO [RS:1;4ab3d709c6df:43401 {}] regionserver.HRegionServer(1031): Exiting; stopping=4ab3d709c6df,43401,1730978016916; zookeeper connection closed. 2024-11-07T11:13:45,127 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2bead503 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2bead503 2024-11-07T11:13:45,127 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-07T11:13:45,146 DEBUG [M:0;4ab3d709c6df:37891 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/27441fbd145844fc8faad89bc5a1bcde is 69, key is 4ab3d709c6df,33951,1730978016967/rs:state/1730978018326/Put/seqid=0 2024-11-07T11:13:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741873_1049 (size=5447) 2024-11-07T11:13:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741873_1049 (size=5447) 2024-11-07T11:13:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741873_1049 (size=5447) 2024-11-07T11:13:45,154 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/27441fbd145844fc8faad89bc5a1bcde 2024-11-07T11:13:45,160 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 27441fbd145844fc8faad89bc5a1bcde 2024-11-07T11:13:45,181 DEBUG [M:0;4ab3d709c6df:37891 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f10b7e09acf2439e8eb5d31c78910b36 is 52, key is load_balancer_on/state:d/1730978021878/Put/seqid=0 2024-11-07T11:13:45,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741874_1050 (size=5058) 2024-11-07T11:13:45,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741874_1050 (size=5058) 2024-11-07T11:13:45,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741874_1050 (size=5058) 2024-11-07T11:13:45,189 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f10b7e09acf2439e8eb5d31c78910b36 2024-11-07T11:13:45,196 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b66b7b61ceb740209792c4bf904fef81 as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b66b7b61ceb740209792c4bf904fef81 2024-11-07T11:13:45,203 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b66b7b61ceb740209792c4bf904fef81, entries=14, sequenceid=575, filesize=5.9 K 2024-11-07T11:13:45,204 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/90aa978450154cc08e742731ce559969 as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/90aa978450154cc08e742731ce559969 2024-11-07T11:13:45,210 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/90aa978450154cc08e742731ce559969, entries=79, sequenceid=575, filesize=12.5 K 2024-11-07T11:13:45,211 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/27441fbd145844fc8faad89bc5a1bcde as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/27441fbd145844fc8faad89bc5a1bcde 2024-11-07T11:13:45,218 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 27441fbd145844fc8faad89bc5a1bcde 2024-11-07T11:13:45,219 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/27441fbd145844fc8faad89bc5a1bcde, entries=3, sequenceid=575, filesize=5.3 K 2024-11-07T11:13:45,220 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f10b7e09acf2439e8eb5d31c78910b36 as hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f10b7e09acf2439e8eb5d31c78910b36 2024-11-07T11:13:45,227 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37645/user/jenkins/test-data/7a0ff0f0-16b4-1df6-2963-f1fab04752ab/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f10b7e09acf2439e8eb5d31c78910b36, entries=1, sequenceid=575, filesize=4.9 K 2024-11-07T11:13:45,228 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.05 KB/256048, heapSize ~303.83 KB/311120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=575, compaction requested=false 2024-11-07T11:13:45,229 INFO [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:13:45,229 DEBUG [M:0;4ab3d709c6df:37891 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730978025059Disabling compacts and flushes for region at 1730978025059Disabling writes for close at 1730978025059Obtaining lock to block concurrent updates at 1730978025060 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730978025060Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256048, getHeapSize=311120, getOffHeapSize=0, getCellsCount=714 at 1730978025060Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730978025060Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730978025060Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730978025075 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730978025075Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730978025092 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730978025111 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730978025111Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730978025130 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730978025145 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730978025145Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1730978025160 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1730978025180 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1730978025181 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ef672c0: reopening flushed file at 1730978025195 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5389b7f0: reopening flushed file at 1730978025203 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74f8ea2f: reopening flushed file at 1730978025210 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4810928e: reopening flushed file at 1730978025219 (+9 ms)Finished flush of dataSize ~250.05 KB/256048, heapSize ~303.83 KB/311120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=575, compaction requested=false at 1730978025228 (+9 ms)Writing region close event to WAL at 1730978025229 (+1 ms)Closed at 1730978025229 2024-11-07T11:13:45,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45515 is added to blk_1073741830_1006 (size=294432) 2024-11-07T11:13:45,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35325 is added to blk_1073741830_1006 (size=294432) 2024-11-07T11:13:45,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44443 is added to blk_1073741830_1006 (size=294432) 2024-11-07T11:13:45,233 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:13:45,233 INFO [M:0;4ab3d709c6df:37891 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T11:13:45,233 INFO [M:0;4ab3d709c6df:37891 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37891 2024-11-07T11:13:45,234 INFO [M:0;4ab3d709c6df:37891 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:13:45,341 INFO [M:0;4ab3d709c6df:37891 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:13:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37891-0x101150063ff0000, quorum=127.0.0.1:54468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:13:45,373 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T11:13:45,376 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T11:13:45,376 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T11:13:45,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T11:13:45,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,STOPPED} 2024-11-07T11:13:45,380 WARN [BP-1322027227-172.17.0.2-1730978011492 heartbeating to localhost/127.0.0.1:37645 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T11:13:45,380 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T11:13:45,380 WARN [BP-1322027227-172.17.0.2-1730978011492 heartbeating to localhost/127.0.0.1:37645 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1322027227-172.17.0.2-1730978011492 (Datanode Uuid 3a9dbdde-f6ec-4c05-bd5c-a57bcc0cae55) service to localhost/127.0.0.1:37645 2024-11-07T11:13:45,380 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T11:13:45,382 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data5/current/BP-1322027227-172.17.0.2-1730978011492 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T11:13:45,382 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data6/current/BP-1322027227-172.17.0.2-1730978011492 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T11:13:45,382 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T11:13:45,384 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T11:13:45,385 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T11:13:45,385 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T11:13:45,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T11:13:45,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,STOPPED} 2024-11-07T11:13:45,386 WARN [BP-1322027227-172.17.0.2-1730978011492 heartbeating to localhost/127.0.0.1:37645 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T11:13:45,386 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T11:13:45,386 WARN [BP-1322027227-172.17.0.2-1730978011492 heartbeating to localhost/127.0.0.1:37645 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1322027227-172.17.0.2-1730978011492 (Datanode Uuid bc2c6651-5d3f-4923-ba9c-d15aeada7b4c) service to localhost/127.0.0.1:37645 2024-11-07T11:13:45,386 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T11:13:45,387 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data3/current/BP-1322027227-172.17.0.2-1730978011492 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T11:13:45,387 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data4/current/BP-1322027227-172.17.0.2-1730978011492 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T11:13:45,387 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T11:13:45,389 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T11:13:45,389 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T11:13:45,389 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T11:13:45,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T11:13:45,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,STOPPED} 2024-11-07T11:13:45,391 WARN [BP-1322027227-172.17.0.2-1730978011492 heartbeating to localhost/127.0.0.1:37645 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T11:13:45,391 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T11:13:45,391 WARN [BP-1322027227-172.17.0.2-1730978011492 heartbeating to localhost/127.0.0.1:37645 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1322027227-172.17.0.2-1730978011492 (Datanode Uuid d269a19d-938c-44cc-84f8-95f4302f4f7c) service to localhost/127.0.0.1:37645 2024-11-07T11:13:45,391 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T11:13:45,392 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data1/current/BP-1322027227-172.17.0.2-1730978011492 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T11:13:45,392 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/cluster_4d419a22-6d32-a2b7-e4de-8fdb1d14ff25/data/data2/current/BP-1322027227-172.17.0.2-1730978011492 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T11:13:45,392 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T11:13:45,402 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T11:13:45,403 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T11:13:45,403 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T11:13:45,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T11:13:45,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/188b8939-19a2-460e-860d-ed738425ab53/hadoop.log.dir/,STOPPED} 2024-11-07T11:13:45,412 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T11:13:45,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down