2024-11-18 19:12:07,378 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-18 19:12:07,396 main DEBUG Took 0.015733 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 19:12:07,396 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 19:12:07,396 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 19:12:07,397 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 19:12:07,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,405 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 19:12:07,417 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,419 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,420 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,421 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,421 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,422 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,423 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,424 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,424 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,424 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,425 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,426 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,426 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,426 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,427 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,428 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,428 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:12:07,429 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,429 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 19:12:07,430 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:12:07,432 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 19:12:07,433 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 19:12:07,434 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 19:12:07,435 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 19:12:07,435 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 19:12:07,444 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 19:12:07,446 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 19:12:07,448 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 19:12:07,448 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 19:12:07,449 main DEBUG createAppenders(={Console}) 2024-11-18 19:12:07,450 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-18 19:12:07,450 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-18 19:12:07,450 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-18 19:12:07,451 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 19:12:07,451 main DEBUG OutputStream closed 2024-11-18 19:12:07,451 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 19:12:07,452 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 19:12:07,452 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-18 19:12:07,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 19:12:07,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 19:12:07,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 19:12:07,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 19:12:07,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 19:12:07,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 19:12:07,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 19:12:07,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 19:12:07,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 19:12:07,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 19:12:07,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 19:12:07,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 19:12:07,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 19:12:07,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 19:12:07,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 19:12:07,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 19:12:07,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 19:12:07,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 19:12:07,533 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 19:12:07,533 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-18 19:12:07,533 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 19:12:07,534 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-18T19:12:07,546 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-18 19:12:07,549 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 19:12:07,549 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T19:12:07,775 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de 2024-11-18T19:12:07,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T19:12:07,812 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175, deleteOnExit=true 2024-11-18T19:12:07,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T19:12:07,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/test.cache.data in system properties and HBase conf 2024-11-18T19:12:07,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T19:12:07,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/hadoop.log.dir in system properties and HBase conf 2024-11-18T19:12:07,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T19:12:07,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T19:12:07,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T19:12:07,922 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T19:12:08,039 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T19:12:08,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T19:12:08,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T19:12:08,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T19:12:08,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T19:12:08,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T19:12:08,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T19:12:08,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T19:12:08,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T19:12:08,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T19:12:08,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/nfs.dump.dir in system properties and HBase conf 2024-11-18T19:12:08,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/java.io.tmpdir in system properties and HBase conf 2024-11-18T19:12:08,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T19:12:08,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T19:12:08,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T19:12:09,006 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T19:12:09,077 INFO [Time-limited test {}] log.Log(170): Logging initialized @2354ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T19:12:09,143 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:12:09,203 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:12:09,227 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:12:09,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:12:09,229 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T19:12:09,241 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:12:09,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d9452bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:12:09,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d5b118c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T19:12:09,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3260f08c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/java.io.tmpdir/jetty-localhost-35205-hadoop-hdfs-3_4_1-tests_jar-_-any-14524875972108506984/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T19:12:09,427 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@369f03e8{HTTP/1.1, (http/1.1)}{localhost:35205} 2024-11-18T19:12:09,427 INFO [Time-limited test {}] server.Server(415): Started @2705ms 2024-11-18T19:12:09,913 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:12:09,920 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:12:09,921 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:12:09,921 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:12:09,922 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T19:12:09,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20ec9bac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:12:09,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5026a795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T19:12:10,020 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f33137f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/java.io.tmpdir/jetty-localhost-35017-hadoop-hdfs-3_4_1-tests_jar-_-any-5240249454646966533/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T19:12:10,021 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6de9cba2{HTTP/1.1, (http/1.1)}{localhost:35017} 2024-11-18T19:12:10,021 INFO [Time-limited test {}] server.Server(415): Started @3298ms 2024-11-18T19:12:10,068 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T19:12:10,172 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:12:10,177 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:12:10,180 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:12:10,181 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:12:10,181 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T19:12:10,182 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b3caf0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:12:10,182 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540e5e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T19:12:10,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7549e2f1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/java.io.tmpdir/jetty-localhost-35345-hadoop-hdfs-3_4_1-tests_jar-_-any-1922946529980745736/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T19:12:10,292 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53e0679{HTTP/1.1, (http/1.1)}{localhost:35345} 2024-11-18T19:12:10,292 INFO [Time-limited test {}] server.Server(415): Started @3570ms 2024-11-18T19:12:10,294 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T19:12:10,333 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:12:10,338 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:12:10,339 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:12:10,339 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:12:10,340 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T19:12:10,344 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469f5a42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:12:10,344 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75445ba5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T19:12:10,442 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6eecab6b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/java.io.tmpdir/jetty-localhost-40277-hadoop-hdfs-3_4_1-tests_jar-_-any-13015912168795508531/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T19:12:10,443 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fb3fd4b{HTTP/1.1, (http/1.1)}{localhost:40277} 2024-11-18T19:12:10,443 INFO [Time-limited test {}] server.Server(415): Started @3721ms 2024-11-18T19:12:10,445 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T19:12:11,345 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/data/data1/current/BP-125440793-172.17.0.2-1731957128563/current, will proceed with Du for space computation calculation, 2024-11-18T19:12:11,345 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/data/data2/current/BP-125440793-172.17.0.2-1731957128563/current, will proceed with Du for space computation calculation, 2024-11-18T19:12:11,345 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/data/data3/current/BP-125440793-172.17.0.2-1731957128563/current, will proceed with Du for space computation calculation, 2024-11-18T19:12:11,345 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/data/data4/current/BP-125440793-172.17.0.2-1731957128563/current, will proceed with Du for space computation calculation, 2024-11-18T19:12:11,379 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T19:12:11,379 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T19:12:11,422 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/data/data6/current/BP-125440793-172.17.0.2-1731957128563/current, will proceed with Du for space computation calculation, 2024-11-18T19:12:11,422 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/data/data5/current/BP-125440793-172.17.0.2-1731957128563/current, will proceed with Du for space computation calculation, 2024-11-18T19:12:11,425 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f4aba92be95f892 with lease ID 0xaab7e5e7dbafd545: Processing first storage report for DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b from datanode DatanodeRegistration(127.0.0.1:35199, datanodeUuid=7ed477b8-9cbb-43e0-8947-836a080116f8, infoPort=39581, infoSecurePort=0, ipcPort=45311, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563) 2024-11-18T19:12:11,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f4aba92be95f892 with lease ID 0xaab7e5e7dbafd545: from storage DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b node DatanodeRegistration(127.0.0.1:35199, datanodeUuid=7ed477b8-9cbb-43e0-8947-836a080116f8, infoPort=39581, infoSecurePort=0, ipcPort=45311, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T19:12:11,427 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28445938ca69a9ba with lease ID 0xaab7e5e7dbafd544: Processing first storage report for DS-336ed3f1-c96c-4d93-ba0e-868c7431f472 from datanode DatanodeRegistration(127.0.0.1:41665, datanodeUuid=5b3e2ba9-2c2a-45ae-a1fa-3e1b5eeed040, infoPort=34031, infoSecurePort=0, ipcPort=35029, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563) 2024-11-18T19:12:11,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28445938ca69a9ba with lease ID 0xaab7e5e7dbafd544: from storage DS-336ed3f1-c96c-4d93-ba0e-868c7431f472 node DatanodeRegistration(127.0.0.1:41665, datanodeUuid=5b3e2ba9-2c2a-45ae-a1fa-3e1b5eeed040, infoPort=34031, infoSecurePort=0, ipcPort=35029, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T19:12:11,428 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f4aba92be95f892 with lease ID 0xaab7e5e7dbafd545: Processing first storage report for DS-76881f90-e53b-411f-8fe0-addec6dde280 from datanode DatanodeRegistration(127.0.0.1:35199, datanodeUuid=7ed477b8-9cbb-43e0-8947-836a080116f8, infoPort=39581, infoSecurePort=0, ipcPort=45311, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563) 2024-11-18T19:12:11,428 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f4aba92be95f892 with lease ID 0xaab7e5e7dbafd545: from storage DS-76881f90-e53b-411f-8fe0-addec6dde280 node DatanodeRegistration(127.0.0.1:35199, datanodeUuid=7ed477b8-9cbb-43e0-8947-836a080116f8, infoPort=39581, infoSecurePort=0, ipcPort=45311, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T19:12:11,428 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28445938ca69a9ba with lease ID 0xaab7e5e7dbafd544: Processing first storage report for DS-da810380-faa6-45d6-8e71-2738436db578 from datanode DatanodeRegistration(127.0.0.1:41665, datanodeUuid=5b3e2ba9-2c2a-45ae-a1fa-3e1b5eeed040, infoPort=34031, infoSecurePort=0, ipcPort=35029, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563) 2024-11-18T19:12:11,429 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28445938ca69a9ba with lease ID 0xaab7e5e7dbafd544: from storage DS-da810380-faa6-45d6-8e71-2738436db578 node DatanodeRegistration(127.0.0.1:41665, datanodeUuid=5b3e2ba9-2c2a-45ae-a1fa-3e1b5eeed040, infoPort=34031, infoSecurePort=0, ipcPort=35029, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T19:12:11,440 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T19:12:11,444 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb06292ff3fa8c616 with lease ID 0xaab7e5e7dbafd546: Processing first storage report for DS-3b86d384-7933-4541-bda7-0c7b28fa62d0 from datanode DatanodeRegistration(127.0.0.1:35133, datanodeUuid=bba5592b-6696-43b1-9f1a-6a2ee165c65a, infoPort=38063, infoSecurePort=0, ipcPort=35169, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563) 2024-11-18T19:12:11,444 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb06292ff3fa8c616 with lease ID 0xaab7e5e7dbafd546: from storage DS-3b86d384-7933-4541-bda7-0c7b28fa62d0 node DatanodeRegistration(127.0.0.1:35133, datanodeUuid=bba5592b-6696-43b1-9f1a-6a2ee165c65a, infoPort=38063, infoSecurePort=0, ipcPort=35169, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T19:12:11,444 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb06292ff3fa8c616 with lease ID 0xaab7e5e7dbafd546: Processing first storage report for DS-8f0f67cb-b2fb-4269-b229-cf706c27c59e from datanode DatanodeRegistration(127.0.0.1:35133, datanodeUuid=bba5592b-6696-43b1-9f1a-6a2ee165c65a, infoPort=38063, infoSecurePort=0, ipcPort=35169, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563) 2024-11-18T19:12:11,445 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb06292ff3fa8c616 with lease ID 0xaab7e5e7dbafd546: from storage DS-8f0f67cb-b2fb-4269-b229-cf706c27c59e node DatanodeRegistration(127.0.0.1:35133, datanodeUuid=bba5592b-6696-43b1-9f1a-6a2ee165c65a, infoPort=38063, infoSecurePort=0, ipcPort=35169, storageInfo=lv=-57;cid=testClusterID;nsid=490912723;c=1731957128563), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T19:12:11,563 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de 2024-11-18T19:12:11,625 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/zookeeper_0, clientPort=49195, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d6ba0435-095d-0cb3-9046-a8cfdc65c2de/cluster_429f0ce9-c8be-5844-9b4f-7963d20d8175/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T19:12:11,634 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49195 2024-11-18T19:12:11,644 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:11,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:11,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741825_1001 (size=7) 2024-11-18T19:12:11,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741825_1001 (size=7) 2024-11-18T19:12:11,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741825_1001 (size=7) 2024-11-18T19:12:12,272 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 with version=8 2024-11-18T19:12:12,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/hbase-staging 2024-11-18T19:12:12,351 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T19:12:12,547 INFO [Time-limited test {}] client.ConnectionUtils(128): master/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T19:12:12,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:12,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:12,561 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T19:12:12,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:12,562 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T19:12:12,685 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T19:12:12,735 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T19:12:12,743 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T19:12:12,746 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T19:12:12,767 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 99927 (auto-detected) 2024-11-18T19:12:12,768 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T19:12:12,786 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39457 2024-11-18T19:12:12,806 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39457 connecting to ZooKeeper ensemble=127.0.0.1:49195 2024-11-18T19:12:12,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394570x0, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T19:12:12,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39457-0x10150a63b420000 connected 2024-11-18T19:12:12,975 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:12,978 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:12,988 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:12,992 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53, hbase.cluster.distributed=false 2024-11-18T19:12:13,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T19:12:13,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39457 2024-11-18T19:12:13,021 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39457 2024-11-18T19:12:13,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39457 2024-11-18T19:12:13,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39457 2024-11-18T19:12:13,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39457 2024-11-18T19:12:13,118 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T19:12:13,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,120 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T19:12:13,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T19:12:13,123 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T19:12:13,125 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T19:12:13,126 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38801 2024-11-18T19:12:13,128 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38801 connecting to ZooKeeper ensemble=127.0.0.1:49195 2024-11-18T19:12:13,129 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,134 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388010x0, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T19:12:13,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388010x0, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:13,149 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38801-0x10150a63b420001 connected 2024-11-18T19:12:13,153 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T19:12:13,161 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T19:12:13,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T19:12:13,169 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T19:12:13,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38801 2024-11-18T19:12:13,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38801 2024-11-18T19:12:13,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38801 2024-11-18T19:12:13,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38801 2024-11-18T19:12:13,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38801 2024-11-18T19:12:13,190 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T19:12:13,190 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,190 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,191 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T19:12:13,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T19:12:13,191 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T19:12:13,191 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T19:12:13,192 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36767 2024-11-18T19:12:13,194 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36767 connecting to ZooKeeper ensemble=127.0.0.1:49195 2024-11-18T19:12:13,195 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,198 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367670x0, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T19:12:13,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:367670x0, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:13,215 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36767-0x10150a63b420002 connected 2024-11-18T19:12:13,216 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T19:12:13,217 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T19:12:13,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T19:12:13,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T19:12:13,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36767 2024-11-18T19:12:13,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36767 2024-11-18T19:12:13,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36767 2024-11-18T19:12:13,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36767 2024-11-18T19:12:13,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36767 2024-11-18T19:12:13,240 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T19:12:13,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,241 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T19:12:13,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:12:13,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T19:12:13,241 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T19:12:13,242 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T19:12:13,242 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39103 2024-11-18T19:12:13,244 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39103 connecting to ZooKeeper ensemble=127.0.0.1:49195 2024-11-18T19:12:13,245 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,248 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391030x0, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T19:12:13,257 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39103-0x10150a63b420003 connected 2024-11-18T19:12:13,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:13,258 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T19:12:13,259 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T19:12:13,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T19:12:13,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T19:12:13,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39103 2024-11-18T19:12:13,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39103 2024-11-18T19:12:13,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39103 2024-11-18T19:12:13,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39103 2024-11-18T19:12:13,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39103 2024-11-18T19:12:13,282 DEBUG [M:0;39fff3b0f89c:39457 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;39fff3b0f89c:39457 2024-11-18T19:12:13,283 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:13,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,301 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,324 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T19:12:13,325 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/39fff3b0f89c,39457,1731957132397 from backup master directory 2024-11-18T19:12:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T19:12:13,332 WARN [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T19:12:13,333 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:13,335 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T19:12:13,336 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T19:12:13,393 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/hbase.id] with ID: 36bffa65-ad57-45e9-886f-853e0e6c4ad1 2024-11-18T19:12:13,393 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/.tmp/hbase.id 2024-11-18T19:12:13,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741826_1002 (size=42) 2024-11-18T19:12:13,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741826_1002 (size=42) 2024-11-18T19:12:13,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741826_1002 (size=42) 2024-11-18T19:12:13,408 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/.tmp/hbase.id]:[hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/hbase.id] 2024-11-18T19:12:13,450 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:12:13,454 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T19:12:13,471 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-18T19:12:13,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:13,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741827_1003 (size=196) 2024-11-18T19:12:13,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741827_1003 (size=196) 2024-11-18T19:12:13,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741827_1003 (size=196) 2024-11-18T19:12:13,517 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:12:13,518 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T19:12:13,531 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:13,535 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:12:13,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741828_1004 (size=1189) 2024-11-18T19:12:13,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741828_1004 (size=1189) 2024-11-18T19:12:13,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741828_1004 (size=1189) 2024-11-18T19:12:13,581 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store 2024-11-18T19:12:13,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741829_1005 (size=34) 2024-11-18T19:12:13,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741829_1005 (size=34) 2024-11-18T19:12:13,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741829_1005 (size=34) 2024-11-18T19:12:13,609 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T19:12:13,613 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:13,614 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T19:12:13,614 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:13,615 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:13,616 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T19:12:13,617 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:13,617 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:13,618 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731957133614Disabling compacts and flushes for region at 1731957133614Disabling writes for close at 1731957133617 (+3 ms)Writing region close event to WAL at 1731957133617Closed at 1731957133617 2024-11-18T19:12:13,620 WARN [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/.initializing 2024-11-18T19:12:13,621 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/WALs/39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:13,629 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T19:12:13,643 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C39457%2C1731957132397, suffix=, logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/WALs/39fff3b0f89c,39457,1731957132397, archiveDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/oldWALs, maxLogs=10 2024-11-18T19:12:13,672 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/WALs/39fff3b0f89c,39457,1731957132397/39fff3b0f89c%2C39457%2C1731957132397.1731957133647, exclude list is [], retry=0 2024-11-18T19:12:13,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41665,DS-336ed3f1-c96c-4d93-ba0e-868c7431f472,DISK] 2024-11-18T19:12:13,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35199,DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b,DISK] 2024-11-18T19:12:13,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35133,DS-3b86d384-7933-4541-bda7-0c7b28fa62d0,DISK] 2024-11-18T19:12:13,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T19:12:13,727 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/WALs/39fff3b0f89c,39457,1731957132397/39fff3b0f89c%2C39457%2C1731957132397.1731957133647 2024-11-18T19:12:13,728 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38063:38063),(127.0.0.1/127.0.0.1:39581:39581),(127.0.0.1/127.0.0.1:34031:34031)] 2024-11-18T19:12:13,729 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:12:13,729 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:13,732 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,733 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T19:12:13,796 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:13,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:13,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,802 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T19:12:13,803 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:13,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:13,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T19:12:13,807 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:13,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:13,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T19:12:13,812 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:13,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:13,814 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,817 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,818 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,825 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,826 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,830 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:12:13,836 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:12:13,841 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:13,842 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72101843, jitterRate=0.07440118491649628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:12:13,849 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731957133744Initializing all the Stores at 1731957133746 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957133747 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957133748 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957133748Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957133748Cleaning up temporary data from old regions at 1731957133826 (+78 ms)Region opened successfully at 1731957133849 (+23 ms) 2024-11-18T19:12:13,851 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T19:12:13,883 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d3d6739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T19:12:13,910 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T19:12:13,919 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:13,919 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:13,922 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:13,923 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T19:12:13,928 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:12:13,928 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T19:12:13,952 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T19:12:13,959 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T19:12:13,998 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T19:12:14,001 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T19:12:14,002 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T19:12:14,006 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T19:12:14,008 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T19:12:14,012 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T19:12:14,022 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T19:12:14,024 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T19:12:14,031 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T19:12:14,052 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T19:12:14,056 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T19:12:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,067 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=39fff3b0f89c,39457,1731957132397, sessionid=0x10150a63b420000, setting cluster-up flag (Was=false) 2024-11-18T19:12:14,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,115 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T19:12:14,119 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:14,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,165 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T19:12:14,169 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:14,178 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T19:12:14,245 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:14,253 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T19:12:14,259 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T19:12:14,264 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 39fff3b0f89c,39457,1731957132397 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T19:12:14,269 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(746): ClusterId : 36bffa65-ad57-45e9-886f-853e0e6c4ad1 2024-11-18T19:12:14,269 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(746): ClusterId : 36bffa65-ad57-45e9-886f-853e0e6c4ad1 2024-11-18T19:12:14,271 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:12:14,271 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:12:14,271 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:12:14,271 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(746): ClusterId : 36bffa65-ad57-45e9-886f-853e0e6c4ad1 2024-11-18T19:12:14,271 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:12:14,271 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/39fff3b0f89c:0, corePoolSize=10, maxPoolSize=10 2024-11-18T19:12:14,272 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,272 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T19:12:14,272 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T19:12:14,272 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T19:12:14,272 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T19:12:14,272 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,273 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957164273 2024-11-18T19:12:14,274 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T19:12:14,275 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T19:12:14,278 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:14,278 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T19:12:14,278 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T19:12:14,279 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T19:12:14,279 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T19:12:14,279 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T19:12:14,280 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,284 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T19:12:14,284 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T19:12:14,284 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T19:12:14,284 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T19:12:14,284 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T19:12:14,284 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T19:12:14,285 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T19:12:14,285 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T19:12:14,286 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T19:12:14,286 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:14,286 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T19:12:14,287 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T19:12:14,288 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T19:12:14,290 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731957134289,5,FailOnTimeoutGroup] 2024-11-18T19:12:14,294 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731957134290,5,FailOnTimeoutGroup] 2024-11-18T19:12:14,294 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,295 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T19:12:14,296 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,297 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,299 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T19:12:14,299 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T19:12:14,299 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T19:12:14,299 DEBUG [RS:0;39fff3b0f89c:38801 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f141d30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T19:12:14,299 DEBUG [RS:1;39fff3b0f89c:36767 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b91213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T19:12:14,299 DEBUG [RS:2;39fff3b0f89c:39103 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7f57cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T19:12:14,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741831_1007 (size=1321) 2024-11-18T19:12:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741831_1007 (size=1321) 2024-11-18T19:12:14,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741831_1007 (size=1321) 2024-11-18T19:12:14,318 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T19:12:14,318 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:14,323 DEBUG [RS:0;39fff3b0f89c:38801 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;39fff3b0f89c:38801 2024-11-18T19:12:14,323 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;39fff3b0f89c:39103 2024-11-18T19:12:14,326 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;39fff3b0f89c:36767 2024-11-18T19:12:14,326 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T19:12:14,326 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T19:12:14,326 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T19:12:14,327 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T19:12:14,327 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T19:12:14,327 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T19:12:14,327 DEBUG [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T19:12:14,327 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T19:12:14,327 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T19:12:14,330 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,39457,1731957132397 with port=38801, startcode=1731957133087 2024-11-18T19:12:14,330 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,39457,1731957132397 with port=39103, startcode=1731957133240 2024-11-18T19:12:14,330 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,39457,1731957132397 with port=36767, startcode=1731957133189 2024-11-18T19:12:14,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741832_1008 (size=32) 2024-11-18T19:12:14,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741832_1008 (size=32) 2024-11-18T19:12:14,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741832_1008 (size=32) 2024-11-18T19:12:14,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:14,343 DEBUG [RS:0;39fff3b0f89c:38801 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T19:12:14,343 DEBUG [RS:2;39fff3b0f89c:39103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T19:12:14,343 DEBUG [RS:1;39fff3b0f89c:36767 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T19:12:14,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T19:12:14,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T19:12:14,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:14,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:14,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T19:12:14,354 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T19:12:14,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:14,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:14,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T19:12:14,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T19:12:14,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:14,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:14,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T19:12:14,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T19:12:14,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:14,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:14,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T19:12:14,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740 2024-11-18T19:12:14,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740 2024-11-18T19:12:14,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T19:12:14,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T19:12:14,375 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:12:14,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T19:12:14,384 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:14,385 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36651, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T19:12:14,385 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50531, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T19:12:14,385 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T19:12:14,386 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64769728, jitterRate=-0.03485584259033203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:12:14,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731957134343Initializing all the Stores at 1731957134345 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957134345Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957134345Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957134345Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957134345Cleaning up temporary data from old regions at 1731957134374 (+29 ms)Region opened successfully at 1731957134391 (+17 ms) 2024-11-18T19:12:14,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T19:12:14,392 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T19:12:14,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T19:12:14,392 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T19:12:14,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T19:12:14,394 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T19:12:14,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731957134392Disabling compacts and flushes for region at 1731957134392Disabling writes for close at 1731957134392Writing region close event to WAL at 1731957134393 (+1 ms)Closed at 1731957134394 (+1 ms) 2024-11-18T19:12:14,395 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,399 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:14,399 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T19:12:14,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T19:12:14,407 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:14,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:14,411 DEBUG [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:14,412 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:14,412 DEBUG [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42667 2024-11-18T19:12:14,412 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:14,412 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:14,412 DEBUG [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T19:12:14,412 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42667 2024-11-18T19:12:14,412 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T19:12:14,416 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:14,416 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42667 2024-11-18T19:12:14,416 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T19:12:14,421 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:14,424 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T19:12:14,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T19:12:14,456 DEBUG [RS:1;39fff3b0f89c:36767 {}] zookeeper.ZKUtil(111): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:14,456 DEBUG [RS:0;39fff3b0f89c:38801 {}] zookeeper.ZKUtil(111): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,457 WARN [RS:1;39fff3b0f89c:36767 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T19:12:14,457 WARN [RS:0;39fff3b0f89c:38801 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T19:12:14,457 INFO [RS:0;39fff3b0f89c:38801 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:12:14,457 INFO [RS:1;39fff3b0f89c:36767 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:12:14,457 DEBUG [RS:2;39fff3b0f89c:39103 {}] zookeeper.ZKUtil(111): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:14,457 WARN [RS:2;39fff3b0f89c:39103 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T19:12:14,457 DEBUG [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,457 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:14,457 INFO [RS:2;39fff3b0f89c:39103 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:12:14,457 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:14,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,38801,1731957133087] 2024-11-18T19:12:14,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,36767,1731957133189] 2024-11-18T19:12:14,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,39103,1731957133240] 2024-11-18T19:12:14,485 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T19:12:14,485 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T19:12:14,485 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T19:12:14,503 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T19:12:14,503 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T19:12:14,503 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T19:12:14,508 INFO [RS:0;39fff3b0f89c:38801 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T19:12:14,508 INFO [RS:2;39fff3b0f89c:39103 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T19:12:14,508 INFO [RS:1;39fff3b0f89c:36767 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T19:12:14,509 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,509 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,509 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,510 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T19:12:14,510 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T19:12:14,510 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T19:12:14,515 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T19:12:14,515 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T19:12:14,515 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T19:12:14,517 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,517 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,517 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,517 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,517 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,517 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,517 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,517 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,517 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T19:12:14,518 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,518 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:12:14,519 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:12:14,519 DEBUG [RS:0;39fff3b0f89c:38801 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:12:14,519 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:2;39fff3b0f89c:39103 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:12:14,519 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:12:14,519 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:12:14,519 DEBUG [RS:1;39fff3b0f89c:36767 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:12:14,521 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38801,1731957133087-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,521 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,522 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,522 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,522 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,522 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39103,1731957133240-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T19:12:14,524 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,524 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,524 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,524 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,525 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,525 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,36767,1731957133189-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T19:12:14,542 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T19:12:14,542 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T19:12:14,542 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T19:12:14,544 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39103,1731957133240-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,544 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,36767,1731957133189-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,544 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38801,1731957133087-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,544 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,544 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,544 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,544 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.Replication(171): 39fff3b0f89c,39103,1731957133240 started 2024-11-18T19:12:14,544 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.Replication(171): 39fff3b0f89c,36767,1731957133189 started 2024-11-18T19:12:14,544 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.Replication(171): 39fff3b0f89c,38801,1731957133087 started 2024-11-18T19:12:14,561 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,562 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,562 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,39103,1731957133240, RpcServer on 39fff3b0f89c/172.17.0.2:39103, sessionid=0x10150a63b420003 2024-11-18T19:12:14,562 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,36767,1731957133189, RpcServer on 39fff3b0f89c/172.17.0.2:36767, sessionid=0x10150a63b420002 2024-11-18T19:12:14,563 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T19:12:14,563 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T19:12:14,563 DEBUG [RS:1;39fff3b0f89c:36767 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:14,563 DEBUG [RS:2;39fff3b0f89c:39103 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:14,563 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,39103,1731957133240' 2024-11-18T19:12:14,563 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,36767,1731957133189' 2024-11-18T19:12:14,563 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T19:12:14,563 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T19:12:14,564 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T19:12:14,564 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T19:12:14,564 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T19:12:14,565 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T19:12:14,565 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T19:12:14,565 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T19:12:14,565 DEBUG [RS:2;39fff3b0f89c:39103 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:14,565 DEBUG [RS:1;39fff3b0f89c:36767 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:14,565 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,39103,1731957133240' 2024-11-18T19:12:14,565 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,36767,1731957133189' 2024-11-18T19:12:14,565 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T19:12:14,565 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T19:12:14,565 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T19:12:14,565 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T19:12:14,566 DEBUG [RS:1;39fff3b0f89c:36767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T19:12:14,566 DEBUG [RS:2;39fff3b0f89c:39103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T19:12:14,566 INFO [RS:1;39fff3b0f89c:36767 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T19:12:14,566 INFO [RS:2;39fff3b0f89c:39103 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T19:12:14,566 INFO [RS:1;39fff3b0f89c:36767 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T19:12:14,566 INFO [RS:2;39fff3b0f89c:39103 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T19:12:14,567 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:14,567 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,38801,1731957133087, RpcServer on 39fff3b0f89c/172.17.0.2:38801, sessionid=0x10150a63b420001 2024-11-18T19:12:14,567 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T19:12:14,568 DEBUG [RS:0;39fff3b0f89c:38801 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,568 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,38801,1731957133087' 2024-11-18T19:12:14,568 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T19:12:14,568 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T19:12:14,569 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T19:12:14,569 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T19:12:14,569 DEBUG [RS:0;39fff3b0f89c:38801 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,569 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,38801,1731957133087' 2024-11-18T19:12:14,569 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T19:12:14,570 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T19:12:14,571 DEBUG [RS:0;39fff3b0f89c:38801 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T19:12:14,571 INFO [RS:0;39fff3b0f89c:38801 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T19:12:14,571 INFO [RS:0;39fff3b0f89c:38801 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T19:12:14,575 WARN [39fff3b0f89c:39457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T19:12:14,675 INFO [RS:0;39fff3b0f89c:38801 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T19:12:14,675 INFO [RS:1;39fff3b0f89c:36767 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T19:12:14,675 INFO [RS:2;39fff3b0f89c:39103 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T19:12:14,679 INFO [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C36767%2C1731957133189, suffix=, logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189, archiveDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs, maxLogs=32 2024-11-18T19:12:14,680 INFO [RS:0;39fff3b0f89c:38801 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C38801%2C1731957133087, suffix=, logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087, archiveDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs, maxLogs=32 2024-11-18T19:12:14,680 INFO [RS:2;39fff3b0f89c:39103 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C39103%2C1731957133240, suffix=, logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,39103,1731957133240, archiveDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs, maxLogs=32 2024-11-18T19:12:14,696 DEBUG [RS:1;39fff3b0f89c:36767 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189/39fff3b0f89c%2C36767%2C1731957133189.1731957134683, exclude list is [], retry=0 2024-11-18T19:12:14,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41665,DS-336ed3f1-c96c-4d93-ba0e-868c7431f472,DISK] 2024-11-18T19:12:14,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35133,DS-3b86d384-7933-4541-bda7-0c7b28fa62d0,DISK] 2024-11-18T19:12:14,728 DEBUG [RS:2;39fff3b0f89c:39103 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,39103,1731957133240/39fff3b0f89c%2C39103%2C1731957133240.1731957134684, exclude list is [], retry=0 2024-11-18T19:12:14,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35199,DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b,DISK] 2024-11-18T19:12:14,732 DEBUG [RS:0;39fff3b0f89c:38801 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087/39fff3b0f89c%2C38801%2C1731957133087.1731957134684, exclude list is [], retry=0 2024-11-18T19:12:14,736 INFO [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189/39fff3b0f89c%2C36767%2C1731957133189.1731957134683 2024-11-18T19:12:14,737 DEBUG [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34031:34031),(127.0.0.1/127.0.0.1:38063:38063),(127.0.0.1/127.0.0.1:39581:39581)] 2024-11-18T19:12:14,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35199,DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b,DISK] 2024-11-18T19:12:14,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35133,DS-3b86d384-7933-4541-bda7-0c7b28fa62d0,DISK] 2024-11-18T19:12:14,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41665,DS-336ed3f1-c96c-4d93-ba0e-868c7431f472,DISK] 2024-11-18T19:12:14,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41665,DS-336ed3f1-c96c-4d93-ba0e-868c7431f472,DISK] 2024-11-18T19:12:14,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35133,DS-3b86d384-7933-4541-bda7-0c7b28fa62d0,DISK] 2024-11-18T19:12:14,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35199,DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b,DISK] 2024-11-18T19:12:14,750 INFO [RS:2;39fff3b0f89c:39103 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,39103,1731957133240/39fff3b0f89c%2C39103%2C1731957133240.1731957134684 2024-11-18T19:12:14,751 DEBUG [RS:2;39fff3b0f89c:39103 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38063:38063),(127.0.0.1/127.0.0.1:39581:39581),(127.0.0.1/127.0.0.1:34031:34031)] 2024-11-18T19:12:14,753 INFO [RS:0;39fff3b0f89c:38801 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 2024-11-18T19:12:14,755 DEBUG [RS:0;39fff3b0f89c:38801 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34031:34031),(127.0.0.1/127.0.0.1:39581:39581),(127.0.0.1/127.0.0.1:38063:38063)] 2024-11-18T19:12:14,829 DEBUG [39fff3b0f89c:39457 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-18T19:12:14,839 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T19:12:14,845 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T19:12:14,845 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T19:12:14,845 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-18T19:12:14,845 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T19:12:14,845 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T19:12:14,845 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T19:12:14,845 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T19:12:14,845 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T19:12:14,846 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T19:12:14,846 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T19:12:14,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:14,858 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,38801,1731957133087, state=OPENING 2024-11-18T19:12:14,898 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T19:12:14,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:14,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:14,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:14,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:14,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:14,911 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:14,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:15,091 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T19:12:15,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T19:12:15,105 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T19:12:15,105 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:12:15,106 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T19:12:15,109 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C38801%2C1731957133087.meta, suffix=.meta, logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087, archiveDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs, maxLogs=32 2024-11-18T19:12:15,124 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, exclude list is [], retry=0 2024-11-18T19:12:15,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41665,DS-336ed3f1-c96c-4d93-ba0e-868c7431f472,DISK] 2024-11-18T19:12:15,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35133,DS-3b86d384-7933-4541-bda7-0c7b28fa62d0,DISK] 2024-11-18T19:12:15,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35199,DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b,DISK] 2024-11-18T19:12:15,131 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta 2024-11-18T19:12:15,131 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34031:34031),(127.0.0.1/127.0.0.1:38063:38063),(127.0.0.1/127.0.0.1:39581:39581)] 2024-11-18T19:12:15,132 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:12:15,134 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T19:12:15,136 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T19:12:15,140 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T19:12:15,143 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T19:12:15,144 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,144 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T19:12:15,144 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T19:12:15,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T19:12:15,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T19:12:15,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:15,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:15,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T19:12:15,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T19:12:15,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:15,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:15,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T19:12:15,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T19:12:15,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:15,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:15,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T19:12:15,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T19:12:15,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:15,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:15,157 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T19:12:15,159 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740 2024-11-18T19:12:15,162 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740 2024-11-18T19:12:15,165 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T19:12:15,165 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T19:12:15,166 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:12:15,169 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T19:12:15,171 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73530154, jitterRate=0.09568467736244202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:12:15,171 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T19:12:15,173 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731957135144Writing region info on filesystem at 1731957135145 (+1 ms)Initializing all the Stores at 1731957135146 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957135146Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957135147 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957135147Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957135147Cleaning up temporary data from old regions at 1731957135165 (+18 ms)Running coprocessor post-open hooks at 1731957135171 (+6 ms)Region opened successfully at 1731957135173 (+2 ms) 2024-11-18T19:12:15,179 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731957135084 2024-11-18T19:12:15,190 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T19:12:15,191 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T19:12:15,192 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:15,195 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,38801,1731957133087, state=OPEN 2024-11-18T19:12:15,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:15,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:15,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:15,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:15,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:15,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:15,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:15,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:15,232 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:15,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T19:12:15,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,38801,1731957133087 in 319 msec 2024-11-18T19:12:15,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T19:12:15,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 836 msec 2024-11-18T19:12:15,250 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:15,250 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T19:12:15,268 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:12:15,269 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1] 2024-11-18T19:12:15,290 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:12:15,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50659, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:12:15,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1060 sec 2024-11-18T19:12:15,315 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731957135315, completionTime=-1 2024-11-18T19:12:15,318 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-18T19:12:15,318 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:15,344 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-18T19:12:15,344 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957195344 2024-11-18T19:12:15,344 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957255344 2024-11-18T19:12:15,345 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-18T19:12:15,346 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:15,352 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39457,1731957132397-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,353 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39457,1731957132397-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,353 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39457,1731957132397-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,354 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-39fff3b0f89c:39457, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,355 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,355 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,361 DEBUG [master/39fff3b0f89c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T19:12:15,382 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.049sec 2024-11-18T19:12:15,384 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T19:12:15,385 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T19:12:15,386 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T19:12:15,387 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T19:12:15,387 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T19:12:15,388 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39457,1731957132397-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T19:12:15,389 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39457,1731957132397-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T19:12:15,394 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T19:12:15,396 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T19:12:15,396 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39457,1731957132397-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:12:15,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670913f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T19:12:15,484 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T19:12:15,484 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T19:12:15,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39fff3b0f89c,39457,-1 for getting cluster id 2024-11-18T19:12:15,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:12:15,500 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '36bffa65-ad57-45e9-886f-853e0e6c4ad1' 2024-11-18T19:12:15,502 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:12:15,502 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "36bffa65-ad57-45e9-886f-853e0e6c4ad1" 2024-11-18T19:12:15,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efead52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T19:12:15,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39fff3b0f89c,39457,-1] 2024-11-18T19:12:15,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:12:15,508 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:15,509 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:12:15,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657cfec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T19:12:15,512 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:12:15,518 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1] 2024-11-18T19:12:15,519 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:12:15,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:12:15,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:15,543 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:12:15,547 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:15,566 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@242664d6 2024-11-18T19:12:15,567 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:12:15,570 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:12:15,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:12:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:15,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:12:15,603 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:15,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-18T19:12:15,607 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:12:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:12:15,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741837_1013 (size=358) 2024-11-18T19:12:15,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741837_1013 (size=358) 2024-11-18T19:12:15,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741837_1013 (size=358) 2024-11-18T19:12:15,629 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 93c84df2c4a5ec3e29ed823586fea37c, NAME => 'test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,630 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 83c16be18ac836cc02b185da0a033819, NAME => 'test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,630 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e2f1bdafdf390ad35de52e2e494af59c, NAME => 'test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741838_1014 (size=45) 2024-11-18T19:12:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741839_1015 (size=42) 2024-11-18T19:12:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741839_1015 (size=42) 2024-11-18T19:12:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741839_1015 (size=42) 2024-11-18T19:12:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741838_1014 (size=45) 2024-11-18T19:12:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741838_1014 (size=45) 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 93c84df2c4a5ec3e29ed823586fea37c, disabling compactions & flushes 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e2f1bdafdf390ad35de52e2e494af59c, disabling compactions & flushes 2024-11-18T19:12:15,652 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:15,652 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. after waiting 0 ms 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. after waiting 0 ms 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:15,652 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:15,652 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 93c84df2c4a5ec3e29ed823586fea37c: Waiting for close lock at 1731957135652Disabling compacts and flushes for region at 1731957135652Disabling writes for close at 1731957135652Writing region close event to WAL at 1731957135652Closed at 1731957135652 2024-11-18T19:12:15,652 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e2f1bdafdf390ad35de52e2e494af59c: Waiting for close lock at 1731957135652Disabling compacts and flushes for region at 1731957135652Disabling writes for close at 1731957135652Writing region close event to WAL at 1731957135652Closed at 1731957135652 2024-11-18T19:12:15,653 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c61a75733b5a3f03b84b6e597527bc5f, NAME => 'test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,654 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04cf4c4723e9b4f78e59ca75fffd9b53, NAME => 'test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741840_1016 (size=45) 2024-11-18T19:12:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741840_1016 (size=45) 2024-11-18T19:12:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741840_1016 (size=45) 2024-11-18T19:12:15,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 83c16be18ac836cc02b185da0a033819, disabling compactions & flushes 2024-11-18T19:12:15,664 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:15,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:15,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. after waiting 0 ms 2024-11-18T19:12:15,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:15,664 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:15,665 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 83c16be18ac836cc02b185da0a033819: Waiting for close lock at 1731957135664Disabling compacts and flushes for region at 1731957135664Disabling writes for close at 1731957135664Writing region close event to WAL at 1731957135664Closed at 1731957135664 2024-11-18T19:12:15,666 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c2c62ceb958ab1d5bfd5cef6659d0890, NAME => 'test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741842_1018 (size=45) 2024-11-18T19:12:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741841_1017 (size=45) 2024-11-18T19:12:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741841_1017 (size=45) 2024-11-18T19:12:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741841_1017 (size=45) 2024-11-18T19:12:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741842_1018 (size=45) 2024-11-18T19:12:15,688 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,688 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c61a75733b5a3f03b84b6e597527bc5f, disabling compactions & flushes 2024-11-18T19:12:15,688 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:15,689 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:15,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741842_1018 (size=45) 2024-11-18T19:12:15,689 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. after waiting 0 ms 2024-11-18T19:12:15,689 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:15,689 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:15,689 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c61a75733b5a3f03b84b6e597527bc5f: Waiting for close lock at 1731957135688Disabling compacts and flushes for region at 1731957135688Disabling writes for close at 1731957135689 (+1 ms)Writing region close event to WAL at 1731957135689Closed at 1731957135689 2024-11-18T19:12:15,689 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6068e34fbd8b0c94ddb14b0b1840e7aa, NAME => 'test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,690 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,690 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 04cf4c4723e9b4f78e59ca75fffd9b53, disabling compactions & flushes 2024-11-18T19:12:15,690 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:15,690 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:15,690 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. after waiting 0 ms 2024-11-18T19:12:15,690 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:15,690 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:15,691 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04cf4c4723e9b4f78e59ca75fffd9b53: Waiting for close lock at 1731957135690Disabling compacts and flushes for region at 1731957135690Disabling writes for close at 1731957135690Writing region close event to WAL at 1731957135690Closed at 1731957135690 2024-11-18T19:12:15,691 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7468e17fba474acfdc38d26fa77dfaa, NAME => 'test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741843_1019 (size=45) 2024-11-18T19:12:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741843_1019 (size=45) 2024-11-18T19:12:15,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741843_1019 (size=45) 2024-11-18T19:12:15,703 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c2c62ceb958ab1d5bfd5cef6659d0890, disabling compactions & flushes 2024-11-18T19:12:15,704 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:15,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:15,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. after waiting 0 ms 2024-11-18T19:12:15,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:15,704 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:15,704 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c2c62ceb958ab1d5bfd5cef6659d0890: Waiting for close lock at 1731957135704Disabling compacts and flushes for region at 1731957135704Disabling writes for close at 1731957135704Writing region close event to WAL at 1731957135704Closed at 1731957135704 2024-11-18T19:12:15,705 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c35b746b07182e41f9cd2f924282671b, NAME => 'test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741845_1021 (size=45) 2024-11-18T19:12:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741844_1020 (size=45) 2024-11-18T19:12:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:12:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741845_1021 (size=45) 2024-11-18T19:12:15,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741845_1021 (size=45) 2024-11-18T19:12:15,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741844_1020 (size=45) 2024-11-18T19:12:15,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741844_1020 (size=45) 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6068e34fbd8b0c94ddb14b0b1840e7aa, disabling compactions & flushes 2024-11-18T19:12:15,728 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. after waiting 0 ms 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:15,728 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e7468e17fba474acfdc38d26fa77dfaa, disabling compactions & flushes 2024-11-18T19:12:15,728 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6068e34fbd8b0c94ddb14b0b1840e7aa: Waiting for close lock at 1731957135728Disabling compacts and flushes for region at 1731957135728Disabling writes for close at 1731957135728Writing region close event to WAL at 1731957135728Closed at 1731957135728 2024-11-18T19:12:15,728 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:15,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:15,729 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 854fa5bf083055851c2de6bfeca153a3, NAME => 'test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. after waiting 0 ms 2024-11-18T19:12:15,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:15,729 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:15,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7468e17fba474acfdc38d26fa77dfaa: Waiting for close lock at 1731957135728Disabling compacts and flushes for region at 1731957135728Disabling writes for close at 1731957135729 (+1 ms)Writing region close event to WAL at 1731957135729Closed at 1731957135729 2024-11-18T19:12:15,730 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 65e1250b53aa09c8dc18cb6f054b8ef3, NAME => 'test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741846_1022 (size=45) 2024-11-18T19:12:15,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741846_1022 (size=45) 2024-11-18T19:12:15,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741846_1022 (size=45) 2024-11-18T19:12:15,742 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,742 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c35b746b07182e41f9cd2f924282671b, disabling compactions & flushes 2024-11-18T19:12:15,742 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:15,743 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:15,743 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. after waiting 0 ms 2024-11-18T19:12:15,743 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:15,743 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:15,743 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c35b746b07182e41f9cd2f924282671b: Waiting for close lock at 1731957135742Disabling compacts and flushes for region at 1731957135742Disabling writes for close at 1731957135743 (+1 ms)Writing region close event to WAL at 1731957135743Closed at 1731957135743 2024-11-18T19:12:15,744 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a8a6f4467f3ce2a74edb9e68108f7c16, NAME => 'test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741847_1023 (size=45) 2024-11-18T19:12:15,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741847_1023 (size=45) 2024-11-18T19:12:15,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741847_1023 (size=45) 2024-11-18T19:12:15,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 854fa5bf083055851c2de6bfeca153a3, disabling compactions & flushes 2024-11-18T19:12:15,757 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:15,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:15,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. after waiting 0 ms 2024-11-18T19:12:15,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:15,757 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:15,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 854fa5bf083055851c2de6bfeca153a3: Waiting for close lock at 1731957135757Disabling compacts and flushes for region at 1731957135757Disabling writes for close at 1731957135757Writing region close event to WAL at 1731957135757Closed at 1731957135757 2024-11-18T19:12:15,758 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5490955be8bafe13acd309a111ae7d57, NAME => 'test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741848_1024 (size=45) 2024-11-18T19:12:15,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741848_1024 (size=45) 2024-11-18T19:12:15,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741848_1024 (size=45) 2024-11-18T19:12:15,772 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,772 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 65e1250b53aa09c8dc18cb6f054b8ef3, disabling compactions & flushes 2024-11-18T19:12:15,772 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:15,772 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:15,773 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. after waiting 0 ms 2024-11-18T19:12:15,773 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:15,773 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:15,773 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 65e1250b53aa09c8dc18cb6f054b8ef3: Waiting for close lock at 1731957135772Disabling compacts and flushes for region at 1731957135772Disabling writes for close at 1731957135773 (+1 ms)Writing region close event to WAL at 1731957135773Closed at 1731957135773 2024-11-18T19:12:15,774 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9edf883e5dfc34b579ff9c2200b10498, NAME => 'test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741850_1026 (size=45) 2024-11-18T19:12:15,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741850_1026 (size=45) 2024-11-18T19:12:15,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741850_1026 (size=45) 2024-11-18T19:12:15,780 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,780 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5490955be8bafe13acd309a111ae7d57, disabling compactions & flushes 2024-11-18T19:12:15,780 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:15,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741849_1025 (size=45) 2024-11-18T19:12:15,781 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:15,781 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. after waiting 0 ms 2024-11-18T19:12:15,781 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:15,781 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:15,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741849_1025 (size=45) 2024-11-18T19:12:15,781 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5490955be8bafe13acd309a111ae7d57: Waiting for close lock at 1731957135780Disabling compacts and flushes for region at 1731957135780Disabling writes for close at 1731957135781 (+1 ms)Writing region close event to WAL at 1731957135781Closed at 1731957135781 2024-11-18T19:12:15,782 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2df6a61387f05ad72aca10670c6c837a, NAME => 'test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741849_1025 (size=45) 2024-11-18T19:12:15,785 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a8a6f4467f3ce2a74edb9e68108f7c16, disabling compactions & flushes 2024-11-18T19:12:15,786 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:15,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:15,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. after waiting 0 ms 2024-11-18T19:12:15,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:15,786 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:15,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a8a6f4467f3ce2a74edb9e68108f7c16: Waiting for close lock at 1731957135785Disabling compacts and flushes for region at 1731957135785Disabling writes for close at 1731957135786 (+1 ms)Writing region close event to WAL at 1731957135786Closed at 1731957135786 2024-11-18T19:12:15,787 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 906bcb0fdf3bd087da5078f40abdb11a, NAME => 'test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741851_1027 (size=45) 2024-11-18T19:12:15,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741851_1027 (size=45) 2024-11-18T19:12:15,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741851_1027 (size=45) 2024-11-18T19:12:15,791 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,791 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9edf883e5dfc34b579ff9c2200b10498, disabling compactions & flushes 2024-11-18T19:12:15,791 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:15,791 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:15,791 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. after waiting 0 ms 2024-11-18T19:12:15,791 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:15,792 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:15,792 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9edf883e5dfc34b579ff9c2200b10498: Waiting for close lock at 1731957135791Disabling compacts and flushes for region at 1731957135791Disabling writes for close at 1731957135791Writing region close event to WAL at 1731957135791Closed at 1731957135791 2024-11-18T19:12:15,792 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5422fc2846edae2febaf8419c8e7557c, NAME => 'test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741852_1028 (size=45) 2024-11-18T19:12:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741852_1028 (size=45) 2024-11-18T19:12:15,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741852_1028 (size=45) 2024-11-18T19:12:15,801 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,801 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2df6a61387f05ad72aca10670c6c837a, disabling compactions & flushes 2024-11-18T19:12:15,801 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:15,801 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:15,801 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. after waiting 0 ms 2024-11-18T19:12:15,802 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:15,802 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:15,802 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2df6a61387f05ad72aca10670c6c837a: Waiting for close lock at 1731957135801Disabling compacts and flushes for region at 1731957135801Disabling writes for close at 1731957135802 (+1 ms)Writing region close event to WAL at 1731957135802Closed at 1731957135802 2024-11-18T19:12:15,802 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => acb619785982ed899beeeef446d69983, NAME => 'test,rrr,1731957135572.acb619785982ed899beeeef446d69983.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741853_1029 (size=45) 2024-11-18T19:12:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741853_1029 (size=45) 2024-11-18T19:12:15,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741853_1029 (size=45) 2024-11-18T19:12:15,810 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,810 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 906bcb0fdf3bd087da5078f40abdb11a, disabling compactions & flushes 2024-11-18T19:12:15,810 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:15,810 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:15,810 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. after waiting 0 ms 2024-11-18T19:12:15,810 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:15,810 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:15,810 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 906bcb0fdf3bd087da5078f40abdb11a: Waiting for close lock at 1731957135810Disabling compacts and flushes for region at 1731957135810Disabling writes for close at 1731957135810Writing region close event to WAL at 1731957135810Closed at 1731957135810 2024-11-18T19:12:15,811 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a1a1d3809d5777735981a521f6cc196d, NAME => 'test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741854_1030 (size=45) 2024-11-18T19:12:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741854_1030 (size=45) 2024-11-18T19:12:15,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741854_1030 (size=45) 2024-11-18T19:12:15,826 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,826 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5422fc2846edae2febaf8419c8e7557c, disabling compactions & flushes 2024-11-18T19:12:15,826 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:15,826 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:15,826 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. after waiting 0 ms 2024-11-18T19:12:15,826 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:15,826 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:15,826 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5422fc2846edae2febaf8419c8e7557c: Waiting for close lock at 1731957135826Disabling compacts and flushes for region at 1731957135826Disabling writes for close at 1731957135826Writing region close event to WAL at 1731957135826Closed at 1731957135826 2024-11-18T19:12:15,827 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07fe0aab39d9c180ecffd1b70f987a4d, NAME => 'test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741855_1031 (size=45) 2024-11-18T19:12:15,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741855_1031 (size=45) 2024-11-18T19:12:15,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741855_1031 (size=45) 2024-11-18T19:12:15,836 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1731957135572.acb619785982ed899beeeef446d69983.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,836 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing acb619785982ed899beeeef446d69983, disabling compactions & flushes 2024-11-18T19:12:15,836 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:15,836 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:15,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731957135572.acb619785982ed899beeeef446d69983. after waiting 0 ms 2024-11-18T19:12:15,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:15,837 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:15,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for acb619785982ed899beeeef446d69983: Waiting for close lock at 1731957135836Disabling compacts and flushes for region at 1731957135836Disabling writes for close at 1731957135837 (+1 ms)Writing region close event to WAL at 1731957135837Closed at 1731957135837 2024-11-18T19:12:15,837 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5288bd571a792583f30de8b9df67e167, NAME => 'test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741856_1032 (size=45) 2024-11-18T19:12:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741856_1032 (size=45) 2024-11-18T19:12:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741856_1032 (size=45) 2024-11-18T19:12:15,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a1a1d3809d5777735981a521f6cc196d, disabling compactions & flushes 2024-11-18T19:12:15,848 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:15,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:15,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. after waiting 0 ms 2024-11-18T19:12:15,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:15,848 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:15,849 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a1a1d3809d5777735981a521f6cc196d: Waiting for close lock at 1731957135848Disabling compacts and flushes for region at 1731957135848Disabling writes for close at 1731957135848Writing region close event to WAL at 1731957135848Closed at 1731957135848 2024-11-18T19:12:15,850 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7059949908a0286d0db393e5fdfea111, NAME => 'test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741857_1033 (size=45) 2024-11-18T19:12:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741858_1034 (size=45) 2024-11-18T19:12:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741857_1033 (size=45) 2024-11-18T19:12:15,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741858_1034 (size=45) 2024-11-18T19:12:15,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741857_1033 (size=45) 2024-11-18T19:12:15,864 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,864 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 07fe0aab39d9c180ecffd1b70f987a4d, disabling compactions & flushes 2024-11-18T19:12:15,864 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:15,864 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:15,864 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. after waiting 0 ms 2024-11-18T19:12:15,864 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:15,864 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:15,864 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07fe0aab39d9c180ecffd1b70f987a4d: Waiting for close lock at 1731957135864Disabling compacts and flushes for region at 1731957135864Disabling writes for close at 1731957135864Writing region close event to WAL at 1731957135864Closed at 1731957135864 2024-11-18T19:12:15,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741858_1034 (size=45) 2024-11-18T19:12:15,865 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0c0ddbbd34300bf264e72f52607add9, NAME => 'test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,865 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,865 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5288bd571a792583f30de8b9df67e167, disabling compactions & flushes 2024-11-18T19:12:15,865 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:15,866 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:15,866 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. after waiting 0 ms 2024-11-18T19:12:15,866 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:15,866 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:15,866 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5288bd571a792583f30de8b9df67e167: Waiting for close lock at 1731957135865Disabling compacts and flushes for region at 1731957135865Disabling writes for close at 1731957135866 (+1 ms)Writing region close event to WAL at 1731957135866Closed at 1731957135866 2024-11-18T19:12:15,867 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => db50e8b6be6adca837ea205e4b8cb8ea, NAME => 'test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741859_1035 (size=45) 2024-11-18T19:12:15,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741859_1035 (size=45) 2024-11-18T19:12:15,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741859_1035 (size=45) 2024-11-18T19:12:15,875 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,875 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7059949908a0286d0db393e5fdfea111, disabling compactions & flushes 2024-11-18T19:12:15,875 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:15,875 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:15,875 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. after waiting 0 ms 2024-11-18T19:12:15,875 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:15,875 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:15,875 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7059949908a0286d0db393e5fdfea111: Waiting for close lock at 1731957135875Disabling compacts and flushes for region at 1731957135875Disabling writes for close at 1731957135875Writing region close event to WAL at 1731957135875Closed at 1731957135875 2024-11-18T19:12:15,876 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0653b2f57087b13166483a696eef069d, NAME => 'test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741860_1036 (size=45) 2024-11-18T19:12:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741860_1036 (size=45) 2024-11-18T19:12:15,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741860_1036 (size=45) 2024-11-18T19:12:15,880 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741861_1037 (size=45) 2024-11-18T19:12:15,880 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a0c0ddbbd34300bf264e72f52607add9, disabling compactions & flushes 2024-11-18T19:12:15,880 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:15,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741861_1037 (size=45) 2024-11-18T19:12:15,880 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:15,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. after waiting 0 ms 2024-11-18T19:12:15,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:15,881 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:15,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0c0ddbbd34300bf264e72f52607add9: Waiting for close lock at 1731957135880Disabling compacts and flushes for region at 1731957135880Disabling writes for close at 1731957135881 (+1 ms)Writing region close event to WAL at 1731957135881Closed at 1731957135881 2024-11-18T19:12:15,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741861_1037 (size=45) 2024-11-18T19:12:15,882 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e840b4726f8ff2dc5bb7ecc4863af340, NAME => 'test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53 2024-11-18T19:12:15,883 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,883 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing db50e8b6be6adca837ea205e4b8cb8ea, disabling compactions & flushes 2024-11-18T19:12:15,883 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:15,883 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:15,883 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. after waiting 0 ms 2024-11-18T19:12:15,883 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:15,883 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:15,884 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for db50e8b6be6adca837ea205e4b8cb8ea: Waiting for close lock at 1731957135883Disabling compacts and flushes for region at 1731957135883Disabling writes for close at 1731957135883Writing region close event to WAL at 1731957135883Closed at 1731957135883 2024-11-18T19:12:15,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741862_1038 (size=45) 2024-11-18T19:12:15,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741862_1038 (size=45) 2024-11-18T19:12:15,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741862_1038 (size=45) 2024-11-18T19:12:15,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741863_1039 (size=42) 2024-11-18T19:12:15,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741863_1039 (size=42) 2024-11-18T19:12:15,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741863_1039 (size=42) 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e840b4726f8ff2dc5bb7ecc4863af340, disabling compactions & flushes 2024-11-18T19:12:15,893 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. after waiting 0 ms 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:15,893 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e840b4726f8ff2dc5bb7ecc4863af340: Waiting for close lock at 1731957135893Disabling compacts and flushes for region at 1731957135893Disabling writes for close at 1731957135893Writing region close event to WAL at 1731957135893Closed at 1731957135893 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:15,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0653b2f57087b13166483a696eef069d, disabling compactions & flushes 2024-11-18T19:12:15,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:15,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:15,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. after waiting 0 ms 2024-11-18T19:12:15,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:15,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:15,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0653b2f57087b13166483a696eef069d: Waiting for close lock at 1731957135893Disabling compacts and flushes for region at 1731957135893Disabling writes for close at 1731957135894 (+1 ms)Writing region close event to WAL at 1731957135894Closed at 1731957135894 2024-11-18T19:12:15,896 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:12:15,901 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,901 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731957135572.acb619785982ed899beeeef446d69983.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,905 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,905 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,905 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,905 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,905 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731957135896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731957135896"}]},"ts":"1731957135896"} 2024-11-18T19:12:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:12:15,946 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-18T19:12:15,948 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:12:15,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731957135948"}]},"ts":"1731957135948"} 2024-11-18T19:12:15,955 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-18T19:12:15,956 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T19:12:15,973 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T19:12:15,973 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T19:12:15,973 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T19:12:15,973 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T19:12:15,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93c84df2c4a5ec3e29ed823586fea37c, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83c16be18ac836cc02b185da0a033819, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c61a75733b5a3f03b84b6e597527bc5f, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c2c62ceb958ab1d5bfd5cef6659d0890, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6068e34fbd8b0c94ddb14b0b1840e7aa, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c35b746b07182e41f9cd2f924282671b, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=854fa5bf083055851c2de6bfeca153a3, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a8a6f4467f3ce2a74edb9e68108f7c16, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5490955be8bafe13acd309a111ae7d57, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2df6a61387f05ad72aca10670c6c837a, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=906bcb0fdf3bd087da5078f40abdb11a, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acb619785982ed899beeeef446d69983, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a1a1d3809d5777735981a521f6cc196d, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5288bd571a792583f30de8b9df67e167, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7059949908a0286d0db393e5fdfea111, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db50e8b6be6adca837ea205e4b8cb8ea, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e840b4726f8ff2dc5bb7ecc4863af340, ASSIGN}] 2024-11-18T19:12:15,985 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e840b4726f8ff2dc5bb7ecc4863af340, ASSIGN 2024-11-18T19:12:15,987 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e840b4726f8ff2dc5bb7ecc4863af340, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:15,988 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:15,990 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:15,991 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db50e8b6be6adca837ea205e4b8cb8ea, ASSIGN 2024-11-18T19:12:15,993 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=db50e8b6be6adca837ea205e4b8cb8ea, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:15,994 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:15,996 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:15,997 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7059949908a0286d0db393e5fdfea111, ASSIGN 2024-11-18T19:12:15,999 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7059949908a0286d0db393e5fdfea111, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,000 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5288bd571a792583f30de8b9df67e167, ASSIGN 2024-11-18T19:12:16,002 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5288bd571a792583f30de8b9df67e167, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,003 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:16,005 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,006 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a1a1d3809d5777735981a521f6cc196d, ASSIGN 2024-11-18T19:12:16,008 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a1a1d3809d5777735981a521f6cc196d, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,009 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acb619785982ed899beeeef446d69983, ASSIGN 2024-11-18T19:12:16,010 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=acb619785982ed899beeeef446d69983, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,012 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:16,013 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,015 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=906bcb0fdf3bd087da5078f40abdb11a, ASSIGN 2024-11-18T19:12:16,017 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=906bcb0fdf3bd087da5078f40abdb11a, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,019 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2df6a61387f05ad72aca10670c6c837a, ASSIGN 2024-11-18T19:12:16,020 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2df6a61387f05ad72aca10670c6c837a, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,022 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:16,023 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,025 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5490955be8bafe13acd309a111ae7d57, ASSIGN 2024-11-18T19:12:16,026 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5490955be8bafe13acd309a111ae7d57, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,027 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a8a6f4467f3ce2a74edb9e68108f7c16, ASSIGN 2024-11-18T19:12:16,029 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a8a6f4467f3ce2a74edb9e68108f7c16, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,030 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:16,031 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,033 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=854fa5bf083055851c2de6bfeca153a3, ASSIGN 2024-11-18T19:12:16,034 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=854fa5bf083055851c2de6bfeca153a3, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,035 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c35b746b07182e41f9cd2f924282671b, ASSIGN 2024-11-18T19:12:16,036 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c35b746b07182e41f9cd2f924282671b, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,038 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, ASSIGN 2024-11-18T19:12:16,039 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,040 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6068e34fbd8b0c94ddb14b0b1840e7aa, ASSIGN 2024-11-18T19:12:16,042 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6068e34fbd8b0c94ddb14b0b1840e7aa, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,043 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c2c62ceb958ab1d5bfd5cef6659d0890, ASSIGN 2024-11-18T19:12:16,045 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c2c62ceb958ab1d5bfd5cef6659d0890, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:16,048 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,049 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c61a75733b5a3f03b84b6e597527bc5f, ASSIGN 2024-11-18T19:12:16,051 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c61a75733b5a3f03b84b6e597527bc5f, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,052 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83c16be18ac836cc02b185da0a033819, ASSIGN 2024-11-18T19:12:16,053 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=83c16be18ac836cc02b185da0a033819, ASSIGN; state=OFFLINE, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:16,055 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:16,056 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN; state=OFFLINE, location=39fff3b0f89c,38801,1731957133087; forceNewPlan=false, retain=false 2024-11-18T19:12:16,057 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93c84df2c4a5ec3e29ed823586fea37c, ASSIGN 2024-11-18T19:12:16,059 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93c84df2c4a5ec3e29ed823586fea37c, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39103,1731957133240; forceNewPlan=false, retain=false 2024-11-18T19:12:16,140 INFO [39fff3b0f89c:39457 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-18T19:12:16,141 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,143 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=7059949908a0286d0db393e5fdfea111, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a1a1d3809d5777735981a521f6cc196d, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e840b4726f8ff2dc5bb7ecc4863af340, ASSIGN because future has completed 2024-11-18T19:12:16,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=854fa5bf083055851c2de6bfeca153a3, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e840b4726f8ff2dc5bb7ecc4863af340, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7059949908a0286d0db393e5fdfea111, ASSIGN because future has completed 2024-11-18T19:12:16,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a1a1d3809d5777735981a521f6cc196d, ASSIGN because future has completed 2024-11-18T19:12:16,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=906bcb0fdf3bd087da5078f40abdb11a, ASSIGN because future has completed 2024-11-18T19:12:16,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 906bcb0fdf3bd087da5078f40abdb11a, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a8a6f4467f3ce2a74edb9e68108f7c16, ASSIGN because future has completed 2024-11-18T19:12:16,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=854fa5bf083055851c2de6bfeca153a3, ASSIGN because future has completed 2024-11-18T19:12:16,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 854fa5bf083055851c2de6bfeca153a3, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8a6f4467f3ce2a74edb9e68108f7c16, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1a1d3809d5777735981a521f6cc196d, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7059949908a0286d0db393e5fdfea111, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,188 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,189 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0653b2f57087b13166483a696eef069d, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6068e34fbd8b0c94ddb14b0b1840e7aa, ASSIGN because future has completed 2024-11-18T19:12:16,192 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6068e34fbd8b0c94ddb14b0b1840e7aa, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c61a75733b5a3f03b84b6e597527bc5f, ASSIGN because future has completed 2024-11-18T19:12:16,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93c84df2c4a5ec3e29ed823586fea37c, ASSIGN because future has completed 2024-11-18T19:12:16,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN because future has completed 2024-11-18T19:12:16,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN because future has completed 2024-11-18T19:12:16,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN because future has completed 2024-11-18T19:12:16,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93c84df2c4a5ec3e29ed823586fea37c, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c61a75733b5a3f03b84b6e597527bc5f, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:16,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN because future has completed 2024-11-18T19:12:16,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN because future has completed 2024-11-18T19:12:16,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN because future has completed 2024-11-18T19:12:16,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, ASSIGN because future has completed 2024-11-18T19:12:16,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN because future has completed 2024-11-18T19:12:16,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:12:16,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5288bd571a792583f30de8b9df67e167, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=acb619785982ed899beeeef446d69983, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=2df6a61387f05ad72aca10670c6c837a, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN because future has completed 2024-11-18T19:12:16,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5490955be8bafe13acd309a111ae7d57, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:16,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=db50e8b6be6adca837ea205e4b8cb8ea, ASSIGN because future has completed 2024-11-18T19:12:16,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5288bd571a792583f30de8b9df67e167, ASSIGN because future has completed 2024-11-18T19:12:16,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=acb619785982ed899beeeef446d69983, ASSIGN because future has completed 2024-11-18T19:12:16,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2df6a61387f05ad72aca10670c6c837a, ASSIGN because future has completed 2024-11-18T19:12:16,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df6a61387f05ad72aca10670c6c837a, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5490955be8bafe13acd309a111ae7d57, ASSIGN because future has completed 2024-11-18T19:12:16,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5490955be8bafe13acd309a111ae7d57, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure acb619785982ed899beeeef446d69983, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5288bd571a792583f30de8b9df67e167, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure db50e8b6be6adca837ea205e4b8cb8ea, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=c35b746b07182e41f9cd2f924282671b, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=83c16be18ac836cc02b185da0a033819, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c35b746b07182e41f9cd2f924282671b, ASSIGN because future has completed 2024-11-18T19:12:16,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure c35b746b07182e41f9cd2f924282671b, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c2c62ceb958ab1d5bfd5cef6659d0890, ASSIGN because future has completed 2024-11-18T19:12:16,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=83c16be18ac836cc02b185da0a033819, ASSIGN because future has completed 2024-11-18T19:12:16,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83c16be18ac836cc02b185da0a033819, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,303 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T19:12:16,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T19:12:16,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2c62ceb958ab1d5bfd5cef6659d0890, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:16,312 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:16,312 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 93c84df2c4a5ec3e29ed823586fea37c, NAME => 'test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-18T19:12:16,313 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,313 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,313 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,313 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,315 INFO [StoreOpener-93c84df2c4a5ec3e29ed823586fea37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,317 INFO [StoreOpener-93c84df2c4a5ec3e29ed823586fea37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93c84df2c4a5ec3e29ed823586fea37c columnFamilyName family 2024-11-18T19:12:16,317 DEBUG [StoreOpener-93c84df2c4a5ec3e29ed823586fea37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,318 INFO [StoreOpener-93c84df2c4a5ec3e29ed823586fea37c-1 {}] regionserver.HStore(327): Store=93c84df2c4a5ec3e29ed823586fea37c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,319 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,320 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,321 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,321 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,322 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,324 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,326 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/93c84df2c4a5ec3e29ed823586fea37c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,327 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 93c84df2c4a5ec3e29ed823586fea37c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67939639, jitterRate=0.012379512190818787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,327 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:16,329 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 93c84df2c4a5ec3e29ed823586fea37c: Running coprocessor pre-open hook at 1731957136313Writing region info on filesystem at 1731957136313Initializing all the Stores at 1731957136315 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136315Cleaning up temporary data from old regions at 1731957136322 (+7 ms)Running coprocessor post-open hooks at 1731957136327 (+5 ms)Region opened successfully at 1731957136329 (+2 ms) 2024-11-18T19:12:16,330 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c., pid=41, masterSystemTime=1731957136302 2024-11-18T19:12:16,334 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:16,334 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:16,334 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:16,334 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => a1a1d3809d5777735981a521f6cc196d, NAME => 'test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-18T19:12:16,335 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,335 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,335 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,335 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,338 INFO [StoreOpener-a1a1d3809d5777735981a521f6cc196d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93c84df2c4a5ec3e29ed823586fea37c, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,341 INFO [StoreOpener-a1a1d3809d5777735981a521f6cc196d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1a1d3809d5777735981a521f6cc196d columnFamilyName family 2024-11-18T19:12:16,341 DEBUG [StoreOpener-a1a1d3809d5777735981a521f6cc196d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,342 INFO [StoreOpener-a1a1d3809d5777735981a521f6cc196d-1 {}] regionserver.HStore(327): Store=a1a1d3809d5777735981a521f6cc196d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,342 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,343 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,344 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=5 2024-11-18T19:12:16,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 93c84df2c4a5ec3e29ed823586fea37c, server=39fff3b0f89c,39103,1731957133240 in 120 msec 2024-11-18T19:12:16,344 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,345 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93c84df2c4a5ec3e29ed823586fea37c, ASSIGN in 369 msec 2024-11-18T19:12:16,347 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,350 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a1a1d3809d5777735981a521f6cc196d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,351 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened a1a1d3809d5777735981a521f6cc196d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70731257, jitterRate=0.05397786200046539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,351 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:16,351 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for a1a1d3809d5777735981a521f6cc196d: Running coprocessor pre-open hook at 1731957136335Writing region info on filesystem at 1731957136335Initializing all the Stores at 1731957136337 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136337Cleaning up temporary data from old regions at 1731957136345 (+8 ms)Running coprocessor post-open hooks at 1731957136351 (+6 ms)Region opened successfully at 1731957136351 2024-11-18T19:12:16,352 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d., pid=35, masterSystemTime=1731957136302 2024-11-18T19:12:16,355 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:16,355 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:16,355 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:16,356 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a8a6f4467f3ce2a74edb9e68108f7c16, NAME => 'test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-18T19:12:16,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,356 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,356 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,356 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,356 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1a1d3809d5777735981a521f6cc196d, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,361 INFO [StoreOpener-a8a6f4467f3ce2a74edb9e68108f7c16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,361 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:16,362 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 65e1250b53aa09c8dc18cb6f054b8ef3, NAME => 'test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-18T19:12:16,362 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,363 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,363 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,363 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,363 INFO [StoreOpener-a8a6f4467f3ce2a74edb9e68108f7c16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8a6f4467f3ce2a74edb9e68108f7c16 columnFamilyName family 2024-11-18T19:12:16,364 DEBUG [StoreOpener-a8a6f4467f3ce2a74edb9e68108f7c16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=23 2024-11-18T19:12:16,364 INFO [StoreOpener-a8a6f4467f3ce2a74edb9e68108f7c16-1 {}] regionserver.HStore(327): Store=a8a6f4467f3ce2a74edb9e68108f7c16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure a1a1d3809d5777735981a521f6cc196d, server=39fff3b0f89c,39103,1731957133240 in 184 msec 2024-11-18T19:12:16,364 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,365 INFO [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,366 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,367 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,368 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,368 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a1a1d3809d5777735981a521f6cc196d, ASSIGN in 389 msec 2024-11-18T19:12:16,369 INFO [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65e1250b53aa09c8dc18cb6f054b8ef3 columnFamilyName family 2024-11-18T19:12:16,369 DEBUG [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,370 INFO [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] regionserver.HStore(327): Store=65e1250b53aa09c8dc18cb6f054b8ef3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,370 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,371 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,372 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,372 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,373 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,373 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,374 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a8a6f4467f3ce2a74edb9e68108f7c16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,375 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,376 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a8a6f4467f3ce2a74edb9e68108f7c16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67887146, jitterRate=0.011597305536270142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,376 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:16,376 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a8a6f4467f3ce2a74edb9e68108f7c16: Running coprocessor pre-open hook at 1731957136357Writing region info on filesystem at 1731957136357Initializing all the Stores at 1731957136359 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136359Cleaning up temporary data from old regions at 1731957136368 (+9 ms)Running coprocessor post-open hooks at 1731957136376 (+8 ms)Region opened successfully at 1731957136376 2024-11-18T19:12:16,378 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16., pid=34, masterSystemTime=1731957136302 2024-11-18T19:12:16,379 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/65e1250b53aa09c8dc18cb6f054b8ef3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,380 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 65e1250b53aa09c8dc18cb6f054b8ef3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70321211, jitterRate=0.04786770045757294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,380 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:16,380 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 65e1250b53aa09c8dc18cb6f054b8ef3: Running coprocessor pre-open hook at 1731957136363Writing region info on filesystem at 1731957136363Initializing all the Stores at 1731957136365 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136365Cleaning up temporary data from old regions at 1731957136373 (+8 ms)Running coprocessor post-open hooks at 1731957136380 (+7 ms)Region opened successfully at 1731957136380 2024-11-18T19:12:16,382 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:16,382 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:16,382 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:16,382 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3., pid=46, masterSystemTime=1731957136355 2024-11-18T19:12:16,382 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 854fa5bf083055851c2de6bfeca153a3, NAME => 'test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-18T19:12:16,383 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,383 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,383 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,383 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,383 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,387 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:16,387 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:16,387 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:16,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8a6f4467f3ce2a74edb9e68108f7c16, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,387 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => a0c0ddbbd34300bf264e72f52607add9, NAME => 'test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-18T19:12:16,388 INFO [StoreOpener-854fa5bf083055851c2de6bfeca153a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,388 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,388 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,388 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,388 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,391 INFO [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,391 INFO [StoreOpener-854fa5bf083055851c2de6bfeca153a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 854fa5bf083055851c2de6bfeca153a3 columnFamilyName family 2024-11-18T19:12:16,391 DEBUG [StoreOpener-854fa5bf083055851c2de6bfeca153a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,393 INFO [StoreOpener-854fa5bf083055851c2de6bfeca153a3-1 {}] regionserver.HStore(327): Store=854fa5bf083055851c2de6bfeca153a3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,393 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,396 INFO [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0c0ddbbd34300bf264e72f52607add9 columnFamilyName family 2024-11-18T19:12:16,396 DEBUG [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,396 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,397 INFO [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] regionserver.HStore(327): Store=a0c0ddbbd34300bf264e72f52607add9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,397 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,397 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,398 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,398 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,398 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,399 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=16 2024-11-18T19:12:16,400 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure a8a6f4467f3ce2a74edb9e68108f7c16, server=39fff3b0f89c,39103,1731957133240 in 225 msec 2024-11-18T19:12:16,400 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,402 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,403 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a8a6f4467f3ce2a74edb9e68108f7c16, ASSIGN in 424 msec 2024-11-18T19:12:16,406 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/854fa5bf083055851c2de6bfeca153a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,407 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a0c0ddbbd34300bf264e72f52607add9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,408 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened a0c0ddbbd34300bf264e72f52607add9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68730685, jitterRate=0.0241670161485672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,408 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:16,408 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 854fa5bf083055851c2de6bfeca153a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62872637, jitterRate=-0.06312470138072968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,408 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for a0c0ddbbd34300bf264e72f52607add9: Running coprocessor pre-open hook at 1731957136389Writing region info on filesystem at 1731957136389Initializing all the Stores at 1731957136390 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136390Cleaning up temporary data from old regions at 1731957136400 (+10 ms)Running coprocessor post-open hooks at 1731957136408 (+8 ms)Region opened successfully at 1731957136408 2024-11-18T19:12:16,408 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:16,408 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 854fa5bf083055851c2de6bfeca153a3: Running coprocessor pre-open hook at 1731957136383Writing region info on filesystem at 1731957136383Initializing all the Stores at 1731957136386 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136386Cleaning up temporary data from old regions at 1731957136398 (+12 ms)Running coprocessor post-open hooks at 1731957136408 (+10 ms)Region opened successfully at 1731957136408 2024-11-18T19:12:16,409 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3., pid=33, masterSystemTime=1731957136302 2024-11-18T19:12:16,409 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9., pid=40, masterSystemTime=1731957136355 2024-11-18T19:12:16,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=15 2024-11-18T19:12:16,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,38801,1731957133087 in 152 msec 2024-11-18T19:12:16,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,414 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:16,414 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:16,414 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:16,415 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => e2f1bdafdf390ad35de52e2e494af59c, NAME => 'test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-18T19:12:16,415 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:16,415 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,415 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:16,415 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,415 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:16,415 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,416 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,416 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 6068e34fbd8b0c94ddb14b0b1840e7aa, NAME => 'test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-18T19:12:16,416 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,416 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,416 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,416 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,418 INFO [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,419 INFO [StoreOpener-6068e34fbd8b0c94ddb14b0b1840e7aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 854fa5bf083055851c2de6bfeca153a3, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a0c0ddbbd34300bf264e72f52607add9, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,422 INFO [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2f1bdafdf390ad35de52e2e494af59c columnFamilyName family 2024-11-18T19:12:16,422 INFO [StoreOpener-6068e34fbd8b0c94ddb14b0b1840e7aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6068e34fbd8b0c94ddb14b0b1840e7aa columnFamilyName family 2024-11-18T19:12:16,423 DEBUG [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,423 DEBUG [StoreOpener-6068e34fbd8b0c94ddb14b0b1840e7aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,424 INFO [StoreOpener-6068e34fbd8b0c94ddb14b0b1840e7aa-1 {}] regionserver.HStore(327): Store=6068e34fbd8b0c94ddb14b0b1840e7aa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,424 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,424 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=39fff3b0f89c,39103,1731957133240, table=test, region=854fa5bf083055851c2de6bfeca153a3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T19:12:16,425 INFO [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] regionserver.HStore(327): Store=e2f1bdafdf390ad35de52e2e494af59c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,425 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,425 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,426 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T19:12:16,426 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,426 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,427 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,428 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,428 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57621, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T19:12:16,430 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,430 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,431 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=39fff3b0f89c,38801,1731957133087, table=test, region=a0c0ddbbd34300bf264e72f52607add9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T19:12:16,433 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,435 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=14 2024-11-18T19:12:16,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 854fa5bf083055851c2de6bfeca153a3, server=39fff3b0f89c,39103,1731957133240 in 262 msec 2024-11-18T19:12:16,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=854fa5bf083055851c2de6bfeca153a3, ASSIGN in 460 msec 2024-11-18T19:12:16,441 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:16,442 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 5490955be8bafe13acd309a111ae7d57, NAME => 'test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-18T19:12:16,442 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,442 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,442 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,442 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,444 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/6068e34fbd8b0c94ddb14b0b1840e7aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,444 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e2f1bdafdf390ad35de52e2e494af59c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,445 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened e2f1bdafdf390ad35de52e2e494af59c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61307223, jitterRate=-0.08645118772983551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,445 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 6068e34fbd8b0c94ddb14b0b1840e7aa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73178343, jitterRate=0.09044228494167328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,445 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:16,445 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:16,446 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 6068e34fbd8b0c94ddb14b0b1840e7aa: Running coprocessor pre-open hook at 1731957136416Writing region info on filesystem at 1731957136416Initializing all the Stores at 1731957136418 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136419 (+1 ms)Cleaning up temporary data from old regions at 1731957136430 (+11 ms)Running coprocessor post-open hooks at 1731957136445 (+15 ms)Region opened successfully at 1731957136445 2024-11-18T19:12:16,446 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for e2f1bdafdf390ad35de52e2e494af59c: Running coprocessor pre-open hook at 1731957136416Writing region info on filesystem at 1731957136416Initializing all the Stores at 1731957136417 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136418 (+1 ms)Cleaning up temporary data from old regions at 1731957136428 (+10 ms)Running coprocessor post-open hooks at 1731957136445 (+17 ms)Region opened successfully at 1731957136445 2024-11-18T19:12:16,447 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa., pid=37, masterSystemTime=1731957136302 2024-11-18T19:12:16,447 INFO [StoreOpener-5490955be8bafe13acd309a111ae7d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,447 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c., pid=48, masterSystemTime=1731957136355 2024-11-18T19:12:16,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=27 2024-11-18T19:12:16,449 INFO [StoreOpener-5490955be8bafe13acd309a111ae7d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5490955be8bafe13acd309a111ae7d57 columnFamilyName family 2024-11-18T19:12:16,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,38801,1731957133087 in 229 msec 2024-11-18T19:12:16,450 DEBUG [StoreOpener-5490955be8bafe13acd309a111ae7d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,450 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:16,450 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:16,450 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:16,451 INFO [StoreOpener-5490955be8bafe13acd309a111ae7d57-1 {}] regionserver.HStore(327): Store=5490955be8bafe13acd309a111ae7d57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,451 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,451 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 9edf883e5dfc34b579ff9c2200b10498, NAME => 'test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-18T19:12:16,451 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,451 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,451 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,452 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,452 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:16,452 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,452 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:16,452 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:16,452 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,453 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => c61a75733b5a3f03b84b6e597527bc5f, NAME => 'test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-18T19:12:16,453 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,453 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,453 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,453 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,454 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,455 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,455 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,458 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6068e34fbd8b0c94ddb14b0b1840e7aa, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,464 INFO [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,464 INFO [StoreOpener-c61a75733b5a3f03b84b6e597527bc5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,465 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5490955be8bafe13acd309a111ae7d57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,465 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 5490955be8bafe13acd309a111ae7d57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62010910, jitterRate=-0.07596543431282043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,466 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:16,466 INFO [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9edf883e5dfc34b579ff9c2200b10498 columnFamilyName family 2024-11-18T19:12:16,466 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 5490955be8bafe13acd309a111ae7d57: Running coprocessor pre-open hook at 1731957136443Writing region info on filesystem at 1731957136443Initializing all the Stores at 1731957136445 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136445Cleaning up temporary data from old regions at 1731957136455 (+10 ms)Running coprocessor post-open hooks at 1731957136466 (+11 ms)Region opened successfully at 1731957136466 2024-11-18T19:12:16,466 DEBUG [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=6 2024-11-18T19:12:16,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,38801,1731957133087 in 194 msec 2024-11-18T19:12:16,467 INFO [StoreOpener-c61a75733b5a3f03b84b6e597527bc5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c61a75733b5a3f03b84b6e597527bc5f columnFamilyName family 2024-11-18T19:12:16,467 DEBUG [StoreOpener-c61a75733b5a3f03b84b6e597527bc5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,468 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57., pid=50, masterSystemTime=1731957136426 2024-11-18T19:12:16,468 INFO [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] regionserver.HStore(327): Store=9edf883e5dfc34b579ff9c2200b10498/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,468 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,468 INFO [StoreOpener-c61a75733b5a3f03b84b6e597527bc5f-1 {}] regionserver.HStore(327): Store=c61a75733b5a3f03b84b6e597527bc5f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,468 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,470 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,470 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,470 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,471 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,471 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,472 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN in 491 msec 2024-11-18T19:12:16,472 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:16,472 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:16,472 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,473 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,473 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:16,473 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 2df6a61387f05ad72aca10670c6c837a, NAME => 'test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-18T19:12:16,473 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,473 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,474 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,474 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,474 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,475 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,476 INFO [StoreOpener-2df6a61387f05ad72aca10670c6c837a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,479 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c61a75733b5a3f03b84b6e597527bc5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,479 INFO [StoreOpener-2df6a61387f05ad72aca10670c6c837a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df6a61387f05ad72aca10670c6c837a columnFamilyName family 2024-11-18T19:12:16,480 DEBUG [StoreOpener-2df6a61387f05ad72aca10670c6c837a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5490955be8bafe13acd309a111ae7d57, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,480 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/9edf883e5dfc34b579ff9c2200b10498/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,480 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened c61a75733b5a3f03b84b6e597527bc5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67395588, jitterRate=0.004272520542144775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,481 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:16,481 INFO [StoreOpener-2df6a61387f05ad72aca10670c6c837a-1 {}] regionserver.HStore(327): Store=2df6a61387f05ad72aca10670c6c837a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,481 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for c61a75733b5a3f03b84b6e597527bc5f: Running coprocessor pre-open hook at 1731957136454Writing region info on filesystem at 1731957136454Initializing all the Stores at 1731957136455 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136455Cleaning up temporary data from old regions at 1731957136471 (+16 ms)Running coprocessor post-open hooks at 1731957136481 (+10 ms)Region opened successfully at 1731957136481 2024-11-18T19:12:16,481 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,481 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 9edf883e5dfc34b579ff9c2200b10498; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69990522, jitterRate=0.04294005036354065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,481 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:16,482 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 9edf883e5dfc34b579ff9c2200b10498: Running coprocessor pre-open hook at 1731957136452Writing region info on filesystem at 1731957136452Initializing all the Stores at 1731957136454 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136454Cleaning up temporary data from old regions at 1731957136473 (+19 ms)Running coprocessor post-open hooks at 1731957136482 (+9 ms)Region opened successfully at 1731957136482 2024-11-18T19:12:16,482 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f., pid=42, masterSystemTime=1731957136302 2024-11-18T19:12:16,482 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,483 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,484 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,484 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,484 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498., pid=47, masterSystemTime=1731957136355 2024-11-18T19:12:16,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-18T19:12:16,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 6068e34fbd8b0c94ddb14b0b1840e7aa, server=39fff3b0f89c,39103,1731957133240 in 286 msec 2024-11-18T19:12:16,486 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:16,486 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:16,486 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:16,486 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e840b4726f8ff2dc5bb7ecc4863af340, NAME => 'test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-18T19:12:16,487 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,487 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,487 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,487 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,488 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,488 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:16,488 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:16,489 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:16,489 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 07fe0aab39d9c180ecffd1b70f987a4d, NAME => 'test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-18T19:12:16,490 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,490 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,490 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,490 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,491 INFO [StoreOpener-e840b4726f8ff2dc5bb7ecc4863af340-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c61a75733b5a3f03b84b6e597527bc5f, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,492 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=9edf883e5dfc34b579ff9c2200b10498, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,495 INFO [StoreOpener-e840b4726f8ff2dc5bb7ecc4863af340-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e840b4726f8ff2dc5bb7ecc4863af340 columnFamilyName family 2024-11-18T19:12:16,496 DEBUG [StoreOpener-e840b4726f8ff2dc5bb7ecc4863af340-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,497 INFO [StoreOpener-e840b4726f8ff2dc5bb7ecc4863af340-1 {}] regionserver.HStore(327): Store=e840b4726f8ff2dc5bb7ecc4863af340/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,497 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,499 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=8 2024-11-18T19:12:16,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c61a75733b5a3f03b84b6e597527bc5f, server=39fff3b0f89c,39103,1731957133240 in 268 msec 2024-11-18T19:12:16,499 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,500 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,500 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,501 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/2df6a61387f05ad72aca10670c6c837a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,502 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 2df6a61387f05ad72aca10670c6c837a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67951463, jitterRate=0.012555703520774841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,502 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:16,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c61a75733b5a3f03b84b6e597527bc5f, ASSIGN in 523 msec 2024-11-18T19:12:16,502 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 2df6a61387f05ad72aca10670c6c837a: Running coprocessor pre-open hook at 1731957136474Writing region info on filesystem at 1731957136474Initializing all the Stores at 1731957136475 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136475Cleaning up temporary data from old regions at 1731957136484 (+9 ms)Running coprocessor post-open hooks at 1731957136502 (+18 ms)Region opened successfully at 1731957136502 2024-11-18T19:12:16,503 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a., pid=49, masterSystemTime=1731957136426 2024-11-18T19:12:16,504 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,507 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:16,507 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:16,507 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:16,507 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => c35b746b07182e41f9cd2f924282671b, NAME => 'test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-18T19:12:16,508 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,508 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,508 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,508 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=18 2024-11-18T19:12:16,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,38801,1731957133087 in 246 msec 2024-11-18T19:12:16,511 INFO [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN in 534 msec 2024-11-18T19:12:16,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,514 INFO [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07fe0aab39d9c180ecffd1b70f987a4d columnFamilyName family 2024-11-18T19:12:16,514 DEBUG [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,515 INFO [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] regionserver.HStore(327): Store=07fe0aab39d9c180ecffd1b70f987a4d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,515 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,517 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2df6a61387f05ad72aca10670c6c837a, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,518 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,518 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e840b4726f8ff2dc5bb7ecc4863af340/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,518 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,519 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,519 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e840b4726f8ff2dc5bb7ecc4863af340; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64564729, jitterRate=-0.03791056573390961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,520 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:16,520 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e840b4726f8ff2dc5bb7ecc4863af340: Running coprocessor pre-open hook at 1731957136488Writing region info on filesystem at 1731957136488Initializing all the Stores at 1731957136489 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136489Cleaning up temporary data from old regions at 1731957136500 (+11 ms)Running coprocessor post-open hooks at 1731957136520 (+20 ms)Region opened successfully at 1731957136520 2024-11-18T19:12:16,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6068e34fbd8b0c94ddb14b0b1840e7aa, ASSIGN in 540 msec 2024-11-18T19:12:16,520 INFO [StoreOpener-c35b746b07182e41f9cd2f924282671b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,521 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340., pid=31, masterSystemTime=1731957136302 2024-11-18T19:12:16,522 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,525 INFO [StoreOpener-c35b746b07182e41f9cd2f924282671b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c35b746b07182e41f9cd2f924282671b columnFamilyName family 2024-11-18T19:12:16,526 DEBUG [StoreOpener-c35b746b07182e41f9cd2f924282671b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,527 INFO [StoreOpener-c35b746b07182e41f9cd2f924282671b-1 {}] regionserver.HStore(327): Store=c35b746b07182e41f9cd2f924282671b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,527 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,527 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=test, region=5490955be8bafe13acd309a111ae7d57. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T19:12:16,529 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:16,529 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:16,529 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,529 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:16,529 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 7059949908a0286d0db393e5fdfea111, NAME => 'test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-18T19:12:16,530 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,530 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,530 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,530 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,530 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=19 2024-11-18T19:12:16,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 2df6a61387f05ad72aca10670c6c837a, server=39fff3b0f89c,36767,1731957133189 in 249 msec 2024-11-18T19:12:16,530 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,531 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,534 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2df6a61387f05ad72aca10670c6c837a, ASSIGN in 555 msec 2024-11-18T19:12:16,537 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,537 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/07fe0aab39d9c180ecffd1b70f987a4d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,538 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c35b746b07182e41f9cd2f924282671b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,538 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 07fe0aab39d9c180ecffd1b70f987a4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60668400, jitterRate=-0.09597039222717285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,538 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:16,538 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 07fe0aab39d9c180ecffd1b70f987a4d: Running coprocessor pre-open hook at 1731957136491Writing region info on filesystem at 1731957136491Initializing all the Stores at 1731957136492 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136492Cleaning up temporary data from old regions at 1731957136519 (+27 ms)Running coprocessor post-open hooks at 1731957136538 (+19 ms)Region opened successfully at 1731957136538 2024-11-18T19:12:16,539 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened c35b746b07182e41f9cd2f924282671b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60724595, jitterRate=-0.09513302147388458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,539 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:16,539 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for c35b746b07182e41f9cd2f924282671b: Running coprocessor pre-open hook at 1731957136508Writing region info on filesystem at 1731957136508Initializing all the Stores at 1731957136510 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136510Cleaning up temporary data from old regions at 1731957136531 (+21 ms)Running coprocessor post-open hooks at 1731957136539 (+8 ms)Region opened successfully at 1731957136539 2024-11-18T19:12:16,539 INFO [StoreOpener-7059949908a0286d0db393e5fdfea111-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,541 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d., pid=39, masterSystemTime=1731957136355 2024-11-18T19:12:16,541 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b., pid=54, masterSystemTime=1731957136426 2024-11-18T19:12:16,542 INFO [StoreOpener-7059949908a0286d0db393e5fdfea111-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7059949908a0286d0db393e5fdfea111 columnFamilyName family 2024-11-18T19:12:16,542 DEBUG [StoreOpener-7059949908a0286d0db393e5fdfea111-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e840b4726f8ff2dc5bb7ecc4863af340, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,543 INFO [StoreOpener-7059949908a0286d0db393e5fdfea111-1 {}] regionserver.HStore(327): Store=7059949908a0286d0db393e5fdfea111/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,543 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,545 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:16,545 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:16,545 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:16,545 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 04cf4c4723e9b4f78e59ca75fffd9b53, NAME => 'test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-18T19:12:16,546 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,546 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:16,546 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:16,546 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,546 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:16,546 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,546 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,546 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => c2c62ceb958ab1d5bfd5cef6659d0890, NAME => 'test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-18T19:12:16,547 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,547 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,547 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,547 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,547 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,548 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=17 2024-11-18T19:12:16,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 5490955be8bafe13acd309a111ae7d57, server=39fff3b0f89c,36767,1731957133189 in 266 msec 2024-11-18T19:12:16,549 INFO [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,549 INFO [StoreOpener-c2c62ceb958ab1d5bfd5cef6659d0890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,549 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,549 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,551 INFO [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cf4c4723e9b4f78e59ca75fffd9b53 columnFamilyName family 2024-11-18T19:12:16,552 DEBUG [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,552 INFO [StoreOpener-c2c62ceb958ab1d5bfd5cef6659d0890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2c62ceb958ab1d5bfd5cef6659d0890 columnFamilyName family 2024-11-18T19:12:16,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5490955be8bafe13acd309a111ae7d57, ASSIGN in 573 msec 2024-11-18T19:12:16,552 INFO [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] regionserver.HStore(327): Store=04cf4c4723e9b4f78e59ca75fffd9b53/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,552 DEBUG [StoreOpener-c2c62ceb958ab1d5bfd5cef6659d0890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,552 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,553 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,554 INFO [StoreOpener-c2c62ceb958ab1d5bfd5cef6659d0890-1 {}] regionserver.HStore(327): Store=c2c62ceb958ab1d5bfd5cef6659d0890/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,554 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,555 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,555 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,555 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,556 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,556 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,557 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,557 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,558 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/7059949908a0286d0db393e5fdfea111/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,558 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,558 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure c35b746b07182e41f9cd2f924282671b, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,559 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 7059949908a0286d0db393e5fdfea111; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67859064, jitterRate=0.011178851127624512}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,559 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:16,560 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 7059949908a0286d0db393e5fdfea111: Running coprocessor pre-open hook at 1731957136530Writing region info on filesystem at 1731957136530Initializing all the Stores at 1731957136532 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136532Cleaning up temporary data from old regions at 1731957136549 (+17 ms)Running coprocessor post-open hooks at 1731957136560 (+11 ms)Region opened successfully at 1731957136560 2024-11-18T19:12:16,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,561 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,561 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111., pid=36, masterSystemTime=1731957136302 2024-11-18T19:12:16,563 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,565 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:16,565 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:16,565 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:16,566 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 906bcb0fdf3bd087da5078f40abdb11a, NAME => 'test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-18T19:12:16,566 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,566 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,566 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,566 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,568 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c2c62ceb958ab1d5bfd5cef6659d0890/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,569 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened c2c62ceb958ab1d5bfd5cef6659d0890; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74401182, jitterRate=0.10866400599479675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,570 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:16,570 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for c2c62ceb958ab1d5bfd5cef6659d0890: Running coprocessor pre-open hook at 1731957136547Writing region info on filesystem at 1731957136547Initializing all the Stores at 1731957136548 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136548Cleaning up temporary data from old regions at 1731957136557 (+9 ms)Running coprocessor post-open hooks at 1731957136570 (+13 ms)Region opened successfully at 1731957136570 2024-11-18T19:12:16,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=13 2024-11-18T19:12:16,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure c35b746b07182e41f9cd2f924282671b, server=39fff3b0f89c,36767,1731957133189 in 269 msec 2024-11-18T19:12:16,571 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890., pid=56, masterSystemTime=1731957136426 2024-11-18T19:12:16,572 INFO [StoreOpener-906bcb0fdf3bd087da5078f40abdb11a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,572 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/04cf4c4723e9b4f78e59ca75fffd9b53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,574 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 04cf4c4723e9b4f78e59ca75fffd9b53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61067105, jitterRate=-0.09002922475337982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,574 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:16,574 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 04cf4c4723e9b4f78e59ca75fffd9b53: Running coprocessor pre-open hook at 1731957136546Writing region info on filesystem at 1731957136546Initializing all the Stores at 1731957136548 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136548Cleaning up temporary data from old regions at 1731957136558 (+10 ms)Running coprocessor post-open hooks at 1731957136574 (+16 ms)Region opened successfully at 1731957136574 2024-11-18T19:12:16,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c35b746b07182e41f9cd2f924282671b, ASSIGN in 594 msec 2024-11-18T19:12:16,576 INFO [StoreOpener-906bcb0fdf3bd087da5078f40abdb11a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 906bcb0fdf3bd087da5078f40abdb11a columnFamilyName family 2024-11-18T19:12:16,576 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:16,576 DEBUG [StoreOpener-906bcb0fdf3bd087da5078f40abdb11a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,576 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:16,576 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:16,576 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53., pid=44, masterSystemTime=1731957136355 2024-11-18T19:12:16,576 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => acb619785982ed899beeeef446d69983, NAME => 'test,rrr,1731957135572.acb619785982ed899beeeef446d69983.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-18T19:12:16,576 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,577 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,577 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,rrr,1731957135572.acb619785982ed899beeeef446d69983.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,577 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,577 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,579 INFO [StoreOpener-906bcb0fdf3bd087da5078f40abdb11a-1 {}] regionserver.HStore(327): Store=906bcb0fdf3bd087da5078f40abdb11a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,579 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,579 INFO [StoreOpener-acb619785982ed899beeeef446d69983-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2c62ceb958ab1d5bfd5cef6659d0890, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,581 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:16,581 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:16,581 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,581 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => e7468e17fba474acfdc38d26fa77dfaa, NAME => 'test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,582 INFO [StoreOpener-acb619785982ed899beeeef446d69983-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acb619785982ed899beeeef446d69983 columnFamilyName family 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,582 DEBUG [StoreOpener-acb619785982ed899beeeef446d69983-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,582 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,583 INFO [StoreOpener-acb619785982ed899beeeef446d69983-1 {}] regionserver.HStore(327): Store=acb619785982ed899beeeef446d69983/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,583 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,583 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,584 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,585 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,585 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,586 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,586 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,586 INFO [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,589 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,589 INFO [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7468e17fba474acfdc38d26fa77dfaa columnFamilyName family 2024-11-18T19:12:16,589 DEBUG [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,590 INFO [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] regionserver.HStore(327): Store=e7468e17fba474acfdc38d26fa77dfaa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,590 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,592 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,593 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/acb619785982ed899beeeef446d69983/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,593 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,593 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened acb619785982ed899beeeef446d69983; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65025850, jitterRate=-0.031039327383041382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,593 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acb619785982ed899beeeef446d69983 2024-11-18T19:12:16,593 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,594 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=10 2024-11-18T19:12:16,594 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for acb619785982ed899beeeef446d69983: Running coprocessor pre-open hook at 1731957136577Writing region info on filesystem at 1731957136577Initializing all the Stores at 1731957136579 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136579Cleaning up temporary data from old regions at 1731957136586 (+7 ms)Running coprocessor post-open hooks at 1731957136593 (+7 ms)Region opened successfully at 1731957136593 2024-11-18T19:12:16,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure c2c62ceb958ab1d5bfd5cef6659d0890, server=39fff3b0f89c,36767,1731957133189 in 282 msec 2024-11-18T19:12:16,595 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731957135572.acb619785982ed899beeeef446d69983., pid=51, masterSystemTime=1731957136426 2024-11-18T19:12:16,595 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/906bcb0fdf3bd087da5078f40abdb11a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,596 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 906bcb0fdf3bd087da5078f40abdb11a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72678891, jitterRate=0.08299987018108368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,596 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:16,596 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 906bcb0fdf3bd087da5078f40abdb11a: Running coprocessor pre-open hook at 1731957136566Writing region info on filesystem at 1731957136566Initializing all the Stores at 1731957136568 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136568Cleaning up temporary data from old regions at 1731957136582 (+14 ms)Running coprocessor post-open hooks at 1731957136596 (+14 ms)Region opened successfully at 1731957136596 2024-11-18T19:12:16,597 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a., pid=32, masterSystemTime=1731957136302 2024-11-18T19:12:16,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c2c62ceb958ab1d5bfd5cef6659d0890, ASSIGN in 618 msec 2024-11-18T19:12:16,598 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,599 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:16,599 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:16,599 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:16,599 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 5288bd571a792583f30de8b9df67e167, NAME => 'test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-18T19:12:16,600 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,600 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:16,600 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:16,600 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,601 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,601 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=9 2024-11-18T19:12:16,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,38801,1731957133087 in 355 msec 2024-11-18T19:12:16,604 INFO [StoreOpener-5288bd571a792583f30de8b9df67e167-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,604 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,606 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened e7468e17fba474acfdc38d26fa77dfaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61223220, jitterRate=-0.0877029299736023}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,606 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:16,606 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for e7468e17fba474acfdc38d26fa77dfaa: Running coprocessor pre-open hook at 1731957136583Writing region info on filesystem at 1731957136583Initializing all the Stores at 1731957136585 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136585Cleaning up temporary data from old regions at 1731957136594 (+9 ms)Running coprocessor post-open hooks at 1731957136606 (+12 ms)Region opened successfully at 1731957136606 2024-11-18T19:12:16,607 INFO [StoreOpener-5288bd571a792583f30de8b9df67e167-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5288bd571a792583f30de8b9df67e167 columnFamilyName family 2024-11-18T19:12:16,607 DEBUG [StoreOpener-5288bd571a792583f30de8b9df67e167-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN in 628 msec 2024-11-18T19:12:16,607 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa., pid=45, masterSystemTime=1731957136355 2024-11-18T19:12:16,608 INFO [StoreOpener-5288bd571a792583f30de8b9df67e167-1 {}] regionserver.HStore(327): Store=5288bd571a792583f30de8b9df67e167/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,608 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,608 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,610 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,611 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 906bcb0fdf3bd087da5078f40abdb11a, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,612 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:16,612 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:16,612 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:16,612 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,612 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,612 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 0653b2f57087b13166483a696eef069d, NAME => 'test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-18T19:12:16,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=acb619785982ed899beeeef446d69983, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,613 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,613 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,613 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,613 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure acb619785982ed899beeeef446d69983, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,616 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,617 INFO [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,621 INFO [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0653b2f57087b13166483a696eef069d columnFamilyName family 2024-11-18T19:12:16,621 DEBUG [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-18T19:12:16,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 906bcb0fdf3bd087da5078f40abdb11a, server=39fff3b0f89c,39103,1731957133240 in 461 msec 2024-11-18T19:12:16,622 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5288bd571a792583f30de8b9df67e167/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,622 INFO [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] regionserver.HStore(327): Store=0653b2f57087b13166483a696eef069d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,622 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,623 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 5288bd571a792583f30de8b9df67e167; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63030599, jitterRate=-0.06077088415622711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,623 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:16,624 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 5288bd571a792583f30de8b9df67e167: Running coprocessor pre-open hook at 1731957136601Writing region info on filesystem at 1731957136601Initializing all the Stores at 1731957136603 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136604 (+1 ms)Cleaning up temporary data from old regions at 1731957136612 (+8 ms)Running coprocessor post-open hooks at 1731957136623 (+11 ms)Region opened successfully at 1731957136624 (+1 ms) 2024-11-18T19:12:16,624 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,624 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,625 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167., pid=52, masterSystemTime=1731957136426 2024-11-18T19:12:16,625 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=906bcb0fdf3bd087da5078f40abdb11a, ASSIGN in 646 msec 2024-11-18T19:12:16,625 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,628 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,629 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:16,629 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:16,629 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:16,629 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => db50e8b6be6adca837ea205e4b8cb8ea, NAME => 'test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-18T19:12:16,630 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,630 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,630 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,630 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,632 INFO [StoreOpener-db50e8b6be6adca837ea205e4b8cb8ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,632 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/0653b2f57087b13166483a696eef069d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=22 2024-11-18T19:12:16,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure acb619785982ed899beeeef446d69983, server=39fff3b0f89c,36767,1731957133189 in 345 msec 2024-11-18T19:12:16,633 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 0653b2f57087b13166483a696eef069d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75397720, jitterRate=0.12351357936859131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,633 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:16,633 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 0653b2f57087b13166483a696eef069d: Running coprocessor pre-open hook at 1731957136613Writing region info on filesystem at 1731957136613Initializing all the Stores at 1731957136616 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136616Cleaning up temporary data from old regions at 1731957136625 (+9 ms)Running coprocessor post-open hooks at 1731957136633 (+8 ms)Region opened successfully at 1731957136633 2024-11-18T19:12:16,634 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=39fff3b0f89c,39103,1731957133240, table=test, region=e840b4726f8ff2dc5bb7ecc4863af340. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T19:12:16,635 INFO [StoreOpener-db50e8b6be6adca837ea205e4b8cb8ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db50e8b6be6adca837ea205e4b8cb8ea columnFamilyName family 2024-11-18T19:12:16,635 DEBUG [StoreOpener-db50e8b6be6adca837ea205e4b8cb8ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,635 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731957135572.0653b2f57087b13166483a696eef069d., pid=38, masterSystemTime=1731957136355 2024-11-18T19:12:16,636 INFO [StoreOpener-db50e8b6be6adca837ea205e4b8cb8ea-1 {}] regionserver.HStore(327): Store=db50e8b6be6adca837ea205e4b8cb8ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,637 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=acb619785982ed899beeeef446d69983, ASSIGN in 657 msec 2024-11-18T19:12:16,638 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,639 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,640 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,640 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,640 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:16,640 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:16,640 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:16,640 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=39fff3b0f89c,38801,1731957133087, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T19:12:16,641 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 5422fc2846edae2febaf8419c8e7557c, NAME => 'test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-18T19:12:16,641 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5288bd571a792583f30de8b9df67e167, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,641 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,641 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,641 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,644 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5288bd571a792583f30de8b9df67e167, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0653b2f57087b13166483a696eef069d, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,647 INFO [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,649 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/db50e8b6be6adca837ea205e4b8cb8ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,650 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened db50e8b6be6adca837ea205e4b8cb8ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63938587, jitterRate=-0.047240808606147766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,650 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:16,650 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for db50e8b6be6adca837ea205e4b8cb8ea: Running coprocessor pre-open hook at 1731957136630Writing region info on filesystem at 1731957136630Initializing all the Stores at 1731957136632 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136632Cleaning up temporary data from old regions at 1731957136640 (+8 ms)Running coprocessor post-open hooks at 1731957136650 (+10 ms)Region opened successfully at 1731957136650 2024-11-18T19:12:16,650 INFO [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5422fc2846edae2febaf8419c8e7557c columnFamilyName family 2024-11-18T19:12:16,650 DEBUG [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,651 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea., pid=53, masterSystemTime=1731957136426 2024-11-18T19:12:16,651 INFO [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] regionserver.HStore(327): Store=5422fc2846edae2febaf8419c8e7557c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,652 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,653 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,653 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,654 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,654 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,655 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:16,655 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:16,655 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:16,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=25 2024-11-18T19:12:16,656 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 83c16be18ac836cc02b185da0a033819, NAME => 'test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-18T19:12:16,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 5288bd571a792583f30de8b9df67e167, server=39fff3b0f89c,36767,1731957133189 in 365 msec 2024-11-18T19:12:16,656 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,656 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:16,656 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,656 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,657 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,658 INFO [StoreOpener-83c16be18ac836cc02b185da0a033819-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5288bd571a792583f30de8b9df67e167, ASSIGN in 680 msec 2024-11-18T19:12:16,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,661 INFO [StoreOpener-83c16be18ac836cc02b185da0a033819-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83c16be18ac836cc02b185da0a033819 columnFamilyName family 2024-11-18T19:12:16,661 DEBUG [StoreOpener-83c16be18ac836cc02b185da0a033819-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:16,662 INFO [StoreOpener-83c16be18ac836cc02b185da0a033819-1 {}] regionserver.HStore(327): Store=83c16be18ac836cc02b185da0a033819/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:16,662 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,663 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,664 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure db50e8b6be6adca837ea205e4b8cb8ea, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,665 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,665 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,667 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,668 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5422fc2846edae2febaf8419c8e7557c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=29 2024-11-18T19:12:16,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,38801,1731957133087 in 464 msec 2024-11-18T19:12:16,669 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 5422fc2846edae2febaf8419c8e7557c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64272816, jitterRate=-0.04226040840148926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,669 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:16,669 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 5422fc2846edae2febaf8419c8e7557c: Running coprocessor pre-open hook at 1731957136642Writing region info on filesystem at 1731957136642Initializing all the Stores at 1731957136643 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136643Cleaning up temporary data from old regions at 1731957136654 (+11 ms)Running coprocessor post-open hooks at 1731957136669 (+15 ms)Region opened successfully at 1731957136669 2024-11-18T19:12:16,670 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c., pid=43, masterSystemTime=1731957136355 2024-11-18T19:12:16,671 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/83c16be18ac836cc02b185da0a033819/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:12:16,672 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 83c16be18ac836cc02b185da0a033819; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74445434, jitterRate=0.1093234121799469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:16,672 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:16,672 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 83c16be18ac836cc02b185da0a033819: Running coprocessor pre-open hook at 1731957136657Writing region info on filesystem at 1731957136657Initializing all the Stores at 1731957136658 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957136658Cleaning up temporary data from old regions at 1731957136665 (+7 ms)Running coprocessor post-open hooks at 1731957136672 (+7 ms)Region opened successfully at 1731957136672 2024-11-18T19:12:16,673 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819., pid=55, masterSystemTime=1731957136426 2024-11-18T19:12:16,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN in 693 msec 2024-11-18T19:12:16,674 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:16,674 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:16,676 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:16,676 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:16,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=28 2024-11-18T19:12:16,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure db50e8b6be6adca837ea205e4b8cb8ea, server=39fff3b0f89c,36767,1731957133189 in 385 msec 2024-11-18T19:12:16,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=db50e8b6be6adca837ea205e4b8cb8ea, ASSIGN in 702 msec 2024-11-18T19:12:16,681 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=83c16be18ac836cc02b185da0a033819, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:16,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83c16be18ac836cc02b185da0a033819, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:16,685 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5422fc2846edae2febaf8419c8e7557c, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=7 2024-11-18T19:12:16,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 83c16be18ac836cc02b185da0a033819, server=39fff3b0f89c,36767,1731957133189 in 388 msec 2024-11-18T19:12:16,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=83c16be18ac836cc02b185da0a033819, ASSIGN in 716 msec 2024-11-18T19:12:16,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=21 2024-11-18T19:12:16,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,38801,1731957133087 in 457 msec 2024-11-18T19:12:16,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN in 721 msec 2024-11-18T19:12:16,700 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:16,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:16,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-18T19:12:16,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,38801,1731957133087 in 497 msec 2024-11-18T19:12:16,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN in 731 msec 2024-11-18T19:12:16,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=12 2024-11-18T19:12:16,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,38801,1731957133087 in 462 msec 2024-11-18T19:12:16,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, ASSIGN in 738 msec 2024-11-18T19:12:16,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=7059949908a0286d0db393e5fdfea111, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:16,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7059949908a0286d0db393e5fdfea111, server=39fff3b0f89c,39103,1731957133240 because future has completed 2024-11-18T19:12:16,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-18T19:12:16,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e840b4726f8ff2dc5bb7ecc4863af340, server=39fff3b0f89c,39103,1731957133240 in 574 msec 2024-11-18T19:12:16,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e840b4726f8ff2dc5bb7ecc4863af340, ASSIGN in 748 msec 2024-11-18T19:12:16,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=26 2024-11-18T19:12:16,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 7059949908a0286d0db393e5fdfea111, server=39fff3b0f89c,39103,1731957133240 in 545 msec 2024-11-18T19:12:16,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7059949908a0286d0db393e5fdfea111, ASSIGN in 754 msec 2024-11-18T19:12:16,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN in 757 msec 2024-11-18T19:12:16,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=4 2024-11-18T19:12:16,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN in 760 msec 2024-11-18T19:12:16,742 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:12:16,742 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731957136742"}]},"ts":"1731957136742"} 2024-11-18T19:12:16,746 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-18T19:12:16,748 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:12:16,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.1650 sec 2024-11-18T19:12:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:12:16,757 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-18T19:12:16,757 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-18T19:12:16,758 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:12:16,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-18T19:12:16,781 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:12:16,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-18T19:12:16,782 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:12:16,788 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-18T19:12:16,812 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-18T19:12:16,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:12:16,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:12:16,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:12:16,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:12:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 83c16be18ac836cc02b185da0a033819 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c2c62ceb958ab1d5bfd5cef6659d0890 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c35b746b07182e41f9cd2f924282671b changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5490955be8bafe13acd309a111ae7d57 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2df6a61387f05ad72aca10670c6c837a changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region acb619785982ed899beeeef446d69983 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5288bd571a792583f30de8b9df67e167 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region db50e8b6be6adca837ea205e4b8cb8ea changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e2f1bdafdf390ad35de52e2e494af59c changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 04cf4c4723e9b4f78e59ca75fffd9b53 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e7468e17fba474acfdc38d26fa77dfaa changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 65e1250b53aa09c8dc18cb6f054b8ef3 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9edf883e5dfc34b579ff9c2200b10498 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5422fc2846edae2febaf8419c8e7557c changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 07fe0aab39d9c180ecffd1b70f987a4d changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a0c0ddbbd34300bf264e72f52607add9 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0653b2f57087b13166483a696eef069d changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 93c84df2c4a5ec3e29ed823586fea37c changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c61a75733b5a3f03b84b6e597527bc5f changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6068e34fbd8b0c94ddb14b0b1840e7aa changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 854fa5bf083055851c2de6bfeca153a3 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a8a6f4467f3ce2a74edb9e68108f7c16 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 906bcb0fdf3bd087da5078f40abdb11a changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a1a1d3809d5777735981a521f6cc196d changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7059949908a0286d0db393e5fdfea111 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e840b4726f8ff2dc5bb7ecc4863af340 changed from -1.0 to 0.0, refreshing cache 2024-11-18T19:12:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-18T19:12:16,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-18T19:12:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-18T19:12:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T19:12:16,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T19:12:16,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T19:12:16,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T19:12:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-18T19:12:16,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-18T19:12:16,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-18T19:12:17,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 446 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-18T19:12:17,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-18T19:12:17,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.HMaster(2172): balance hri=e7468e17fba474acfdc38d26fa77dfaa, source=39fff3b0f89c,38801,1731957133087, destination=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:17,366 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:17,368 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e7468e17fba474acfdc38d26fa77dfaa, regionState=CLOSING, regionLocation=39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:17,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE because future has completed 2024-11-18T19:12:17,373 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:12:17,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,38801,1731957133087}] 2024-11-18T19:12:17,534 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,534 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:12:17,535 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing e7468e17fba474acfdc38d26fa77dfaa, disabling compactions & flushes 2024-11-18T19:12:17,535 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,536 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,536 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. after waiting 0 ms 2024-11-18T19:12:17,536 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,544 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:17,547 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,547 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for e7468e17fba474acfdc38d26fa77dfaa: Waiting for close lock at 1731957137535Running coprocessor pre-close hooks at 1731957137535Disabling compacts and flushes for region at 1731957137535Disabling writes for close at 1731957137536 (+1 ms)Writing region close event to WAL at 1731957137537 (+1 ms)Running coprocessor post-close hooks at 1731957137545 (+8 ms)Closed at 1731957137547 (+2 ms) 2024-11-18T19:12:17,547 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding e7468e17fba474acfdc38d26fa77dfaa move to 39fff3b0f89c,36767,1731957133189 record at close sequenceid=2 2024-11-18T19:12:17,551 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e7468e17fba474acfdc38d26fa77dfaa, regionState=CLOSED 2024-11-18T19:12:17,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,38801,1731957133087 because future has completed 2024-11-18T19:12:17,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-18T19:12:17,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,38801,1731957133087 in 181 msec 2024-11-18T19:12:17,558 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE; state=CLOSED, location=39fff3b0f89c,36767,1731957133189; forceNewPlan=false, retain=false 2024-11-18T19:12:17,709 INFO [39fff3b0f89c:39457 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T19:12:17,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:17,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE because future has completed 2024-11-18T19:12:17,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:17,879 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,879 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => e7468e17fba474acfdc38d26fa77dfaa, NAME => 'test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-18T19:12:17,880 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,880 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:17,880 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,880 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,883 INFO [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,884 INFO [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7468e17fba474acfdc38d26fa77dfaa columnFamilyName family 2024-11-18T19:12:17,885 DEBUG [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:17,885 INFO [StoreOpener-e7468e17fba474acfdc38d26fa77dfaa-1 {}] regionserver.HStore(327): Store=e7468e17fba474acfdc38d26fa77dfaa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:17,885 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,886 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,888 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,888 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,888 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,891 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,892 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened e7468e17fba474acfdc38d26fa77dfaa; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73014777, jitterRate=0.0880049616098404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:17,892 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:17,892 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for e7468e17fba474acfdc38d26fa77dfaa: Running coprocessor pre-open hook at 1731957137880Writing region info on filesystem at 1731957137880Initializing all the Stores at 1731957137882 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957137882Cleaning up temporary data from old regions at 1731957137888 (+6 ms)Running coprocessor post-open hooks at 1731957137892 (+4 ms)Region opened successfully at 1731957137892 2024-11-18T19:12:17,893 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa., pid=59, masterSystemTime=1731957137867 2024-11-18T19:12:17,896 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,896 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:17,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, openSeqNum=5, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:17,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,36767,1731957133189 because future has completed 2024-11-18T19:12:17,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-18T19:12:17,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure e7468e17fba474acfdc38d26fa77dfaa, server=39fff3b0f89c,36767,1731957133189 in 187 msec 2024-11-18T19:12:17,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE in 539 msec 2024-11-18T19:12:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-18T19:12:17,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:12:17,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-18T19:12:18,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=385, OpenFileDescriptor=639, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=4608 2024-11-18T19:12:18,003 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-18T19:12:18,004 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-18T19:12:18,004 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:18,005 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 39fff3b0f89c,38801,1731957133087: Simulated kill ***** 2024-11-18T19:12:18,005 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-18T19:12:18,007 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-18T19:12:18,009 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-18T19:12:18,009 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-18T19:12:18,009 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-18T19:12:18,009 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 322882432 }, "NonHeapMemoryUsage": { "committed": 152829952, "init": 7667712, "max": -1, "used": 150532184 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c" } ] } 2024-11-18T19:12:18,012 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39457 {}] master.MasterRpcServices(700): 39fff3b0f89c,38801,1731957133087 reported a fatal error: ***** ABORTING region server 39fff3b0f89c,38801,1731957133087: Simulated kill ***** 2024-11-18T19:12:18,014 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,38801,1731957133087' ***** 2024-11-18T19:12:18,014 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-18T19:12:18,014 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T19:12:18,014 INFO [RS:0;39fff3b0f89c:38801 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-18T19:12:18,014 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T19:12:18,014 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:12:18,014 INFO [RS:0;39fff3b0f89c:38801 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-18T19:12:18,015 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T19:12:18,015 INFO [RS:0;39fff3b0f89c:38801 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;39fff3b0f89c:38801. 2024-11-18T19:12:18,015 DEBUG [RS:0;39fff3b0f89c:38801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:12:18,019 DEBUG [RS:0;39fff3b0f89c:38801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741836_1012 (size=33749) 2024-11-18T19:12:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741836_1012 (size=33749) 2024-11-18T19:12:18,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741836_1012 (size=33749) 2024-11-18T19:12:18,027 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:12:18,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741835_1011 (size=2774) 2024-11-18T19:12:18,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741835_1011 (size=2774) 2024-11-18T19:12:18,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741835_1011 (size=2774) 2024-11-18T19:12:18,030 DEBUG [RS:0;39fff3b0f89c:38801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:18,030 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:12:18,030 INFO [RS:0;39fff3b0f89c:38801 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38801 2024-11-18T19:12:18,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T19:12:18,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,38801,1731957133087 2024-11-18T19:12:18,064 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T19:12:18,073 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,38801,1731957133087] 2024-11-18T19:12:18,081 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,38801,1731957133087 already deleted, retry=false 2024-11-18T19:12:18,081 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 39fff3b0f89c,38801,1731957133087 on 39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:18,087 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:18,092 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:18,093 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 39fff3b0f89c,38801,1731957133087 (carryingMeta=true) 39fff3b0f89c,38801,1731957133087/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@285d1f18[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T19:12:18,095 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true, isMeta: true 2024-11-18T19:12:18,098 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting 2024-11-18T19:12:18,099 INFO [PEWorker-1 {}] master.SplitWALManager(105): 39fff3b0f89c,38801,1731957133087 WAL count=1, meta=true 2024-11-18T19:12:18,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta}] 2024-11-18T19:12:18,109 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:18,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:18,116 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:12:18,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:12:18,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38801-0x10150a63b420001, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:12:18,173 INFO [RS:0;39fff3b0f89c:38801 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T19:12:18,174 INFO [RS:0;39fff3b0f89c:38801 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,38801,1731957133087; zookeeper connection closed. 2024-11-18T19:12:18,174 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@562d891 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@562d891 2024-11-18T19:12:18,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39103 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-18T19:12:18,297 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, size=33.0 K (33749bytes) 2024-11-18T19:12:18,297 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-18T19:12:18,298 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-18T19:12:18,298 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta 2024-11-18T19:12:18,304 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta after 3ms 2024-11-18T19:12:18,313 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta took 18ms 2024-11-18T19:12:18,322 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-18T19:12:18,329 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp 2024-11-18T19:12:18,333 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta so closing down 2024-11-18T19:12:18,333 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T19:12:18,336 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp 2024-11-18T19:12:18,341 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T19:12:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741864_1040 (size=33508) 2024-11-18T19:12:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741864_1040 (size=33508) 2024-11-18T19:12:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741864_1040 (size=33508) 2024-11-18T19:12:18,351 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) 2024-11-18T19:12:18,354 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp to hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-18T19:12:18,357 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-18T19:12:18,357 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, journal: Splitting hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, size=33.0 K (33749bytes) at 1731957138297Finishing writing output for hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta so closing down at 1731957138333 (+36 ms)Creating recovered edits writer path=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp at 1731957138336 (+3 ms)3 split writer threads finished at 1731957138341 (+5 ms)Closed recovered edits writer path=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) at 1731957138351 (+10 ms)Rename recovered edits hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta.temp to hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731957138354 (+3 ms)Processed 62 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731957138357 (+3 ms) 2024-11-18T19:12:18,358 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta 2024-11-18T19:12:18,359 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-18T19:12:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-18T19:12:18,370 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta to hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs 2024-11-18T19:12:18,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-18T19:12:18,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 in 259 msec 2024-11-18T19:12:18,375 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:18,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T19:12:18,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 in 275 msec 2024-11-18T19:12:18,382 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting dir is empty, no logs to split. 2024-11-18T19:12:18,382 INFO [PEWorker-1 {}] master.SplitWALManager(105): 39fff3b0f89c,38801,1731957133087 WAL count=0, meta=true 2024-11-18T19:12:18,382 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 39fff3b0f89c,38801,1731957133087 WAL splitting is done? wals=0, meta=true 2024-11-18T19:12:18,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T19:12:18,386 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:18,388 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:18,521 INFO [regionserver/39fff3b0f89c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T19:12:18,522 INFO [regionserver/39fff3b0f89c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T19:12:18,538 DEBUG [39fff3b0f89c:39457 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T19:12:18,538 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T19:12:18,539 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T19:12:18,539 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T19:12:18,539 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T19:12:18,539 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T19:12:18,539 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T19:12:18,539 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T19:12:18,539 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T19:12:18,540 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:18,541 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPENING 2024-11-18T19:12:18,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:18,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:18,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:18,573 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:18,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:18,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:18,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:18,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:18,734 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T19:12:18,734 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:12:18,734 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T19:12:18,737 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C36767%2C1731957133189.meta, suffix=.meta, logDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189, archiveDir=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs, maxLogs=32 2024-11-18T19:12:18,751 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189/39fff3b0f89c%2C36767%2C1731957133189.meta.1731957138738.meta, exclude list is [], retry=0 2024-11-18T19:12:18,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35199,DS-7f21ddf1-4881-4eb8-bc0e-15c24a50c46b,DISK] 2024-11-18T19:12:18,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35133,DS-3b86d384-7933-4541-bda7-0c7b28fa62d0,DISK] 2024-11-18T19:12:18,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41665,DS-336ed3f1-c96c-4d93-ba0e-868c7431f472,DISK] 2024-11-18T19:12:18,757 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189/39fff3b0f89c%2C36767%2C1731957133189.meta.1731957138738.meta 2024-11-18T19:12:18,758 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39581:39581),(127.0.0.1/127.0.0.1:38063:38063),(127.0.0.1/127.0.0.1:34031:34031)] 2024-11-18T19:12:18,758 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:12:18,758 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T19:12:18,758 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T19:12:18,758 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T19:12:18,758 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T19:12:18,758 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:18,759 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T19:12:18,759 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T19:12:18,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T19:12:18,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T19:12:18,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:18,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:18,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T19:12:18,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T19:12:18,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:18,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:18,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T19:12:18,768 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T19:12:18,768 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:18,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:18,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T19:12:18,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T19:12:18,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:18,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:12:18,771 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T19:12:18,772 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740 2024-11-18T19:12:18,775 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740 2024-11-18T19:12:18,776 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-18T19:12:18,793 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-18T19:12:18,796 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-18T19:12:18,865 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/info/6c6ad119de8b42ed9a4454416ada500b is 138, key is test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c./info:regioninfo/1731957136452/Put/seqid=0 2024-11-18T19:12:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741866_1042 (size=36385) 2024-11-18T19:12:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741866_1042 (size=36385) 2024-11-18T19:12:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741866_1042 (size=36385) 2024-11-18T19:12:18,881 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/info/6c6ad119de8b42ed9a4454416ada500b 2024-11-18T19:12:18,952 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/ns/e9035378bd3d4d5a84d9b7ba3dd52bb6 is 43, key is default/ns:d/1731957135298/Put/seqid=0 2024-11-18T19:12:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741867_1043 (size=5153) 2024-11-18T19:12:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741867_1043 (size=5153) 2024-11-18T19:12:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741867_1043 (size=5153) 2024-11-18T19:12:18,962 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/ns/e9035378bd3d4d5a84d9b7ba3dd52bb6 2024-11-18T19:12:18,987 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/table/01a9d120ad744ad2b118cad9364bbe3a is 40, key is test/table:state/1731957136742/Put/seqid=0 2024-11-18T19:12:18,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741868_1044 (size=5165) 2024-11-18T19:12:18,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741868_1044 (size=5165) 2024-11-18T19:12:18,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741868_1044 (size=5165) 2024-11-18T19:12:18,996 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/table/01a9d120ad744ad2b118cad9364bbe3a 2024-11-18T19:12:19,005 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/info/6c6ad119de8b42ed9a4454416ada500b as hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/info/6c6ad119de8b42ed9a4454416ada500b 2024-11-18T19:12:19,014 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/info/6c6ad119de8b42ed9a4454416ada500b, entries=273, sequenceid=64, filesize=35.5 K 2024-11-18T19:12:19,017 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/ns/e9035378bd3d4d5a84d9b7ba3dd52bb6 as hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/ns/e9035378bd3d4d5a84d9b7ba3dd52bb6 2024-11-18T19:12:19,025 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/ns/e9035378bd3d4d5a84d9b7ba3dd52bb6, entries=2, sequenceid=64, filesize=5.0 K 2024-11-18T19:12:19,027 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/table/01a9d120ad744ad2b118cad9364bbe3a as hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/table/01a9d120ad744ad2b118cad9364bbe3a 2024-11-18T19:12:19,037 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/table/01a9d120ad744ad2b118cad9364bbe3a, entries=2, sequenceid=64, filesize=5.0 K 2024-11-18T19:12:19,041 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 244ms, sequenceid=64, compaction requested=false; wal=null 2024-11-18T19:12:19,042 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T19:12:19,045 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-18T19:12:19,047 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T19:12:19,047 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T19:12:19,048 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:12:19,050 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T19:12:19,053 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-18T19:12:19,054 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71499276, jitterRate=0.06542223691940308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:12:19,054 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T19:12:19,054 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731957138759Writing region info on filesystem at 1731957138759Initializing all the Stores at 1731957138760 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957138760Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957138761 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957138761Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731957138761Obtaining lock to block concurrent updates at 1731957138796 (+35 ms)Preparing flush snapshotting stores in 1588230740 at 1731957138796Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731957138799 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731957138799Flushing 1588230740/info: creating writer at 1731957138800 (+1 ms)Flushing 1588230740/info: appending metadata at 1731957138854 (+54 ms)Flushing 1588230740/info: closing flushed file at 1731957138857 (+3 ms)Flushing 1588230740/ns: creating writer at 1731957138934 (+77 ms)Flushing 1588230740/ns: appending metadata at 1731957138951 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731957138951Flushing 1588230740/table: creating writer at 1731957138970 (+19 ms)Flushing 1588230740/table: appending metadata at 1731957138986 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731957138986Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60042c00: reopening flushed file at 1731957139004 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ae34c42: reopening flushed file at 1731957139016 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bda3765: reopening flushed file at 1731957139026 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 244ms, sequenceid=64, compaction requested=false; wal=null at 1731957139041 (+15 ms)Cleaning up temporary data from old regions at 1731957139047 (+6 ms)Running coprocessor post-open hooks at 1731957139054 (+7 ms)Region opened successfully at 1731957139054 2024-11-18T19:12:19,056 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731957138729 2024-11-18T19:12:19,059 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T19:12:19,059 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T19:12:19,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,061 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:19,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,098 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-18T19:12:19,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 in 525 msec 2024-11-18T19:12:19,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-18T19:12:19,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 717 msec 2024-11-18T19:12:19,104 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 39fff3b0f89c,38801,1731957133087 had 9 regions 2024-11-18T19:12:19,105 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true, isMeta: false 2024-11-18T19:12:19,107 INFO [PEWorker-1 {}] master.SplitWALManager(105): 39fff3b0f89c,38801,1731957133087 WAL count=1, meta=false 2024-11-18T19:12:19,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684}] 2024-11-18T19:12:19,110 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:19,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39103 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-18T19:12:19,280 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684, size=2.7 K (2774bytes) 2024-11-18T19:12:19,280 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 2024-11-18T19:12:19,281 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 after 1ms 2024-11-18T19:12:19,284 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 took 4ms 2024-11-18T19:12:19,287 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 65e1250b53aa09c8dc18cb6f054b8ef3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,288 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a0c0ddbbd34300bf264e72f52607add9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,291 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e2f1bdafdf390ad35de52e2e494af59c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,292 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 9edf883e5dfc34b579ff9c2200b10498: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,294 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 07fe0aab39d9c180ecffd1b70f987a4d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,296 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 04cf4c4723e9b4f78e59ca75fffd9b53: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,298 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e7468e17fba474acfdc38d26fa77dfaa: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,299 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0653b2f57087b13166483a696eef069d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5422fc2846edae2febaf8419c8e7557c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-18T19:12:19,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 so closing down 2024-11-18T19:12:19,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T19:12:19,301 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T19:12:19,301 INFO [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 17 ms; skipped=10; WAL=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-18T19:12:19,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684, journal: Splitting hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684, size=2.7 K (2774bytes) at 1731957139280Finishing writing output for hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 so closing down at 1731957139301 (+21 ms)3 split writer threads finished at 1731957139301Processed 10 edits across 0 Regions in 17 ms; skipped=10; WAL=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1731957139301 2024-11-18T19:12:19,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 2024-11-18T19:12:19,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-18T19:12:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39457 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-18T19:12:19,307 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting/39fff3b0f89c%2C38801%2C1731957133087.1731957134684 to hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs 2024-11-18T19:12:19,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-18T19:12:19,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 in 196 msec 2024-11-18T19:12:19,311 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-18T19:12:19,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 in 203 msec 2024-11-18T19:12:19,316 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,38801,1731957133087-splitting dir is empty, no logs to split. 2024-11-18T19:12:19,316 INFO [PEWorker-1 {}] master.SplitWALManager(105): 39fff3b0f89c,38801,1731957133087 WAL count=0, meta=false 2024-11-18T19:12:19,316 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 39fff3b0f89c,38801,1731957133087 WAL splitting is done? wals=0, meta=false 2024-11-18T19:12:19,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN}] 2024-11-18T19:12:19,322 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,324 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,325 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,326 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,327 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,329 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,330 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,331 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,332 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,333 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,334 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,335 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,336 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,337 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,339 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T19:12:19,474 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T19:12:19,475 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T19:12:19,475 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T19:12:19,475 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T19:12:19,475 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T19:12:19,475 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T19:12:19,475 INFO [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T19:12:19,475 DEBUG [39fff3b0f89c:39457 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T19:12:19,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, regionLocation=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,477 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 39fff3b0f89c:38801 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:19,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=0653b2f57087b13166483a696eef069d, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, regionLocation=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T19:12:19,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 from cache 2024-11-18T19:12:19,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 39fff3b0f89c:38801 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,481 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 39fff3b0f89c:38801 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:19,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,38801,1731957133087, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=39fff3b0f89c:38801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 39fff3b0f89c:38801 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:38801 2024-11-18T19:12:19,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:12:19,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,36767,1731957133189, seqNum=-1] 2024-11-18T19:12:19,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:12:19,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:12:19,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN because future has completed 2024-11-18T19:12:19,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:19,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN because future has completed 2024-11-18T19:12:19,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN because future has completed 2024-11-18T19:12:19,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN because future has completed 2024-11-18T19:12:19,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN because future has completed 2024-11-18T19:12:19,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN because future has completed 2024-11-18T19:12:19,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN because future has completed 2024-11-18T19:12:19,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN because future has completed 2024-11-18T19:12:19,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:19,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:19,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:19,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:19,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240}] 2024-11-18T19:12:19,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:19,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,36767,1731957133189}] 2024-11-18T19:12:19,764 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:19,764 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => e2f1bdafdf390ad35de52e2e494af59c, NAME => 'test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-18T19:12:19,764 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,765 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,765 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,765 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,766 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:19,766 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 04cf4c4723e9b4f78e59ca75fffd9b53, NAME => 'test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-18T19:12:19,767 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,767 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,767 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,767 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,767 INFO [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,769 INFO [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2f1bdafdf390ad35de52e2e494af59c columnFamilyName family 2024-11-18T19:12:19,769 DEBUG [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,769 INFO [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,769 INFO [StoreOpener-e2f1bdafdf390ad35de52e2e494af59c-1 {}] regionserver.HStore(327): Store=e2f1bdafdf390ad35de52e2e494af59c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,769 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,770 INFO [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cf4c4723e9b4f78e59ca75fffd9b53 columnFamilyName family 2024-11-18T19:12:19,770 DEBUG [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,770 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,770 INFO [StoreOpener-04cf4c4723e9b4f78e59ca75fffd9b53-1 {}] regionserver.HStore(327): Store=04cf4c4723e9b4f78e59ca75fffd9b53/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,771 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,772 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,772 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,773 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,773 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,773 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,773 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,774 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,775 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,776 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,776 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened e2f1bdafdf390ad35de52e2e494af59c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70575725, jitterRate=0.05166025459766388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,777 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:19,777 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 04cf4c4723e9b4f78e59ca75fffd9b53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72975011, jitterRate=0.08741240203380585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,777 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:19,777 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for e2f1bdafdf390ad35de52e2e494af59c: Running coprocessor pre-open hook at 1731957139765Writing region info on filesystem at 1731957139765Initializing all the Stores at 1731957139767 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139767Cleaning up temporary data from old regions at 1731957139773 (+6 ms)Running coprocessor post-open hooks at 1731957139777 (+4 ms)Region opened successfully at 1731957139777 2024-11-18T19:12:19,777 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 04cf4c4723e9b4f78e59ca75fffd9b53: Running coprocessor pre-open hook at 1731957139767Writing region info on filesystem at 1731957139767Initializing all the Stores at 1731957139768 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139768Cleaning up temporary data from old regions at 1731957139774 (+6 ms)Running coprocessor post-open hooks at 1731957139777 (+3 ms)Region opened successfully at 1731957139777 2024-11-18T19:12:19,778 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53., pid=81, masterSystemTime=1731957139759 2024-11-18T19:12:19,778 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c., pid=78, masterSystemTime=1731957139756 2024-11-18T19:12:19,780 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:19,780 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:19,781 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:19,781 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 5422fc2846edae2febaf8419c8e7557c, NAME => 'test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-18T19:12:19,781 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,781 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,781 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,781 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,783 INFO [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,783 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-18T19:12:19,784 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,784 INFO [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5422fc2846edae2febaf8419c8e7557c columnFamilyName family 2024-11-18T19:12:19,784 DEBUG [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,785 INFO [StoreOpener-5422fc2846edae2febaf8419c8e7557c-1 {}] regionserver.HStore(327): Store=5422fc2846edae2febaf8419c8e7557c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,785 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,784 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,786 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,787 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,788 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,788 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:19,788 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:19,788 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,788 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:19,788 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 07fe0aab39d9c180ecffd1b70f987a4d, NAME => 'test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-18T19:12:19,789 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,789 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,789 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,789 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,791 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,791 INFO [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,792 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 5422fc2846edae2febaf8419c8e7557c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61484389, jitterRate=-0.0838112086057663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,792 INFO [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07fe0aab39d9c180ecffd1b70f987a4d columnFamilyName family 2024-11-18T19:12:19,792 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:19,792 DEBUG [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,793 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 5422fc2846edae2febaf8419c8e7557c: Running coprocessor pre-open hook at 1731957139781Writing region info on filesystem at 1731957139781Initializing all the Stores at 1731957139783 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139783Cleaning up temporary data from old regions at 1731957139788 (+5 ms)Running coprocessor post-open hooks at 1731957139792 (+4 ms)Region opened successfully at 1731957139792 2024-11-18T19:12:19,793 INFO [StoreOpener-07fe0aab39d9c180ecffd1b70f987a4d-1 {}] regionserver.HStore(327): Store=07fe0aab39d9c180ecffd1b70f987a4d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,793 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,794 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c., pid=80, masterSystemTime=1731957139756 2024-11-18T19:12:19,794 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,796 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,797 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,797 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,797 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:19,798 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:19,798 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:19,798 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => a0c0ddbbd34300bf264e72f52607add9, NAME => 'test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-18T19:12:19,798 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,798 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,799 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,799 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,798 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-18T19:12:19,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:19,800 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,800 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,36767,1731957133189 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,801 INFO [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,802 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 07fe0aab39d9c180ecffd1b70f987a4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69657122, jitterRate=0.03797200322151184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,802 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:19,802 INFO [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0c0ddbbd34300bf264e72f52607add9 columnFamilyName family 2024-11-18T19:12:19,802 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 07fe0aab39d9c180ecffd1b70f987a4d: Running coprocessor pre-open hook at 1731957139789Writing region info on filesystem at 1731957139789Initializing all the Stores at 1731957139790 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139790Cleaning up temporary data from old regions at 1731957139797 (+7 ms)Running coprocessor post-open hooks at 1731957139802 (+5 ms)Region opened successfully at 1731957139802 2024-11-18T19:12:19,802 DEBUG [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,803 INFO [StoreOpener-a0c0ddbbd34300bf264e72f52607add9-1 {}] regionserver.HStore(327): Store=a0c0ddbbd34300bf264e72f52607add9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,803 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,804 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d., pid=76, masterSystemTime=1731957139759 2024-11-18T19:12:19,804 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,806 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,807 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,807 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,807 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:19,807 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:19,807 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. 2024-11-18T19:12:19,807 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 0653b2f57087b13166483a696eef069d, NAME => 'test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-18T19:12:19,808 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,808 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,yyy,1731957135572.0653b2f57087b13166483a696eef069d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,808 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,808 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,809 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,809 INFO [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,810 INFO [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0653b2f57087b13166483a696eef069d columnFamilyName family 2024-11-18T19:12:19,810 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened a0c0ddbbd34300bf264e72f52607add9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70515034, jitterRate=0.05075588822364807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,810 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:19,810 DEBUG [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,810 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for a0c0ddbbd34300bf264e72f52607add9: Running coprocessor pre-open hook at 1731957139799Writing region info on filesystem at 1731957139799Initializing all the Stores at 1731957139800 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139800Cleaning up temporary data from old regions at 1731957139807 (+7 ms)Running coprocessor post-open hooks at 1731957139810 (+3 ms)Region opened successfully at 1731957139810 2024-11-18T19:12:19,811 INFO [StoreOpener-0653b2f57087b13166483a696eef069d-1 {}] regionserver.HStore(327): Store=0653b2f57087b13166483a696eef069d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,811 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,811 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9., pid=75, masterSystemTime=1731957139756 2024-11-18T19:12:19,812 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,814 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:19,814 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:19,814 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. 2024-11-18T19:12:19,814 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,814 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 65e1250b53aa09c8dc18cb6f054b8ef3, NAME => 'test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-18T19:12:19,815 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,815 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,815 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,815 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:19,815 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,815 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,817 INFO [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,817 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,818 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 0653b2f57087b13166483a696eef069d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70611102, jitterRate=0.05218741297721863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,818 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0653b2f57087b13166483a696eef069d 2024-11-18T19:12:19,818 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 0653b2f57087b13166483a696eef069d: Running coprocessor pre-open hook at 1731957139808Writing region info on filesystem at 1731957139808Initializing all the Stores at 1731957139809 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139809Cleaning up temporary data from old regions at 1731957139815 (+6 ms)Running coprocessor post-open hooks at 1731957139818 (+3 ms)Region opened successfully at 1731957139818 2024-11-18T19:12:19,820 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:19,820 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,820 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731957135572.0653b2f57087b13166483a696eef069d., pid=77, masterSystemTime=1731957139759 2024-11-18T19:12:19,820 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:19,820 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:19,821 INFO [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65e1250b53aa09c8dc18cb6f054b8ef3 columnFamilyName family 2024-11-18T19:12:19,821 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,821 DEBUG [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:19,821 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:19,821 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:19,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:19,821 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:19,822 INFO [StoreOpener-65e1250b53aa09c8dc18cb6f054b8ef3-1 {}] regionserver.HStore(327): Store=65e1250b53aa09c8dc18cb6f054b8ef3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:19,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:19,822 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,826 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,828 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,829 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,829 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,830 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:19,833 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 65e1250b53aa09c8dc18cb6f054b8ef3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62025057, jitterRate=-0.07575462758541107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:19,833 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65e1250b53aa09c8dc18cb6f054b8ef3 2024-11-18T19:12:19,834 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 65e1250b53aa09c8dc18cb6f054b8ef3: Running coprocessor pre-open hook at 1731957139815Writing region info on filesystem at 1731957139815Initializing all the Stores at 1731957139816 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957139816Cleaning up temporary data from old regions at 1731957139829 (+13 ms)Running coprocessor post-open hooks at 1731957139833 (+4 ms)Region opened successfully at 1731957139834 (+1 ms) 2024-11-18T19:12:19,835 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3., pid=79, masterSystemTime=1731957139756 2024-11-18T19:12:19,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:19,836 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,843 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,845 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,845 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,845 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,845 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9edf883e5dfc34b579ff9c2200b10498, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,846 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-11-18T19:12:19,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:19,848 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:19,849 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:19,849 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "39fff3b0f89c" port: 36767 start_code: 1731957133189 } transition { transition_code: OPENED region_info { region_id: 1731957135572 table_name { namespace: "default" qualifier: "test" } start_key: "yyy" end_key: "zzz" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1731957133333 }; retry (#0) after 1009ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-18T19:12:19,851 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "39fff3b0f89c" port: 39103 start_code: 1731957133240 } transition { transition_code: OPENED region_info { region_id: 1731957135572 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1731957133333 }; retry (#0) after 1003ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-18T19:12:19,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,864 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:19,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:19,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:19,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:19,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:19,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:19,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:19,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:19,867 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957169867 2024-11-18T19:12:19,867 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:19,867 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,870 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:19,871 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:19,871 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,871 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:19,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,909 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:19,910 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957199909 2024-11-18T19:12:19,910 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957259910 2024-11-18T19:12:19,910 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 42 msec 2024-11-18T19:12:19,910 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:19,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,910 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:19,910 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:19,910 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:19,910 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:19,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:19,911 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:19,911 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:19,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:19,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:19,917 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:19,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cf4c4723e9b4f78e59ca75fffd9b53, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,923 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-18T19:12:19,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:19,924 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:19,924 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:19,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,931 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,931 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,931 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,931 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:19,932 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957169932 2024-11-18T19:12:19,932 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:19,932 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,934 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:19,936 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:19,936 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,936 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,944 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:19,944 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957199944 2024-11-18T19:12:19,945 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957259945 2024-11-18T19:12:19,945 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-18T19:12:19,945 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:19,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,945 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:19,945 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:19,945 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:19,945 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:19,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:19,946 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:19,946 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:19,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:19,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:19,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,957 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5422fc2846edae2febaf8419c8e7557c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-18T19:12:19,958 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:19,958 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:19,958 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:19,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:19,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:19,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:19,966 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957169966 2024-11-18T19:12:19,966 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:19,966 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,968 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:19,969 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:19,969 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,969 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:19,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,974 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:19,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:19,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:19,979 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:19,979 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957199979 2024-11-18T19:12:19,979 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957259979 2024-11-18T19:12:19,979 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-18T19:12:19,979 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:19,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:19,979 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:19,980 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:19,980 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:19,980 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:19,980 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:19,980 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:19,981 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:19,986 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:19,987 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:19,988 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:19,988 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:19,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:19,989 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,994 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:19,995 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,995 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65e1250b53aa09c8dc18cb6f054b8ef3, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:19,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-18T19:12:19,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:19,997 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:19,997 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,006 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,006 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:20,007 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170007 2024-11-18T19:12:20,007 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,007 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,009 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,010 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,010 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,011 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,019 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,019 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200019 2024-11-18T19:12:20,019 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260019 2024-11-18T19:12:20,019 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-18T19:12:20,019 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,019 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,019 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,020 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,020 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,020 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,020 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,021 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,028 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,029 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,029 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,029 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,030 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,030 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-18T19:12:20,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,031 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,032 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:20,041 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170041 2024-11-18T19:12:20,041 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,044 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,054 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200054 2024-11-18T19:12:20,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260054 2024-11-18T19:12:20,054 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-18T19:12:20,054 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,054 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,054 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,054 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,054 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,054 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,055 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,064 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,064 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0653b2f57087b13166483a696eef069d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-18T19:12:20,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,066 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,066 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,071 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:20,074 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170074 2024-11-18T19:12:20,074 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,074 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,076 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,077 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,085 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200085 2024-11-18T19:12:20,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260085 2024-11-18T19:12:20,086 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-18T19:12:20,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,086 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,086 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,086 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,087 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,096 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07fe0aab39d9c180ecffd1b70f987a4d, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-18T19:12:20,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,097 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,097 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:20,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170107 2024-11-18T19:12:20,107 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,108 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,109 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,111 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,111 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,111 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,117 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,117 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200117 2024-11-18T19:12:20,117 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260117 2024-11-18T19:12:20,117 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-18T19:12:20,117 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,117 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,118 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,118 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,118 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,118 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,118 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,118 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,119 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,123 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,129 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,129 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0c0ddbbd34300bf264e72f52607add9, server=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-18T19:12:20,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,130 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,130 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:20,141 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170141 2024-11-18T19:12:20,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,141 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,142 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,143 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,151 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200151 2024-11-18T19:12:20,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260151 2024-11-18T19:12:20,151 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-18T19:12:20,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,152 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,152 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,152 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,152 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN 2024-11-18T19:12:20,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T19:12:20,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,160 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,165 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53, ASSIGN to state=OFFLINE, location=null, table=test, region=04cf4c4723e9b4f78e59ca75fffd9b53 to restore RIT 2024-11-18T19:12:20,166 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170166 2024-11-18T19:12:20,166 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,166 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,167 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,168 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,168 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,168 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,175 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200175 2024-11-18T19:12:20,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260175 2024-11-18T19:12:20,175 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-18T19:12:20,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,175 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,180 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,180 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN 2024-11-18T19:12:20,183 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-18T19:12:20,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,184 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,185 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,190 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e2f1bdafdf390ad35de52e2e494af59c, ASSIGN to state=OFFLINE, location=null, table=test, region=e2f1bdafdf390ad35de52e2e494af59c to restore RIT 2024-11-18T19:12:20,191 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170191 2024-11-18T19:12:20,191 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,191 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,192 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,195 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,195 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,195 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,200 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200200 2024-11-18T19:12:20,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260200 2024-11-18T19:12:20,200 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-18T19:12:20,200 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,200 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,200 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,200 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,200 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,200 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,205 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN 2024-11-18T19:12:20,206 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:12:20,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,207 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,207 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9edf883e5dfc34b579ff9c2200b10498, ASSIGN to state=OFFLINE, location=null, table=test, region=9edf883e5dfc34b579ff9c2200b10498 to restore RIT 2024-11-18T19:12:20,216 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170216 2024-11-18T19:12:20,216 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,216 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,220 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,223 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200223 2024-11-18T19:12:20,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260223 2024-11-18T19:12:20,224 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,224 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,224 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,224 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,224 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,224 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,224 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,224 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,224 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,229 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,229 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,229 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN 2024-11-18T19:12:20,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:12:20,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,231 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=65e1250b53aa09c8dc18cb6f054b8ef3 to restore RIT 2024-11-18T19:12:20,240 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170240 2024-11-18T19:12:20,241 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,241 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,242 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,243 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,243 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,244 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,250 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200250 2024-11-18T19:12:20,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260250 2024-11-18T19:12:20,250 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-18T19:12:20,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,251 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,251 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,251 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,251 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,251 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,252 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,257 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,258 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN 2024-11-18T19:12:20,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T19:12:20,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,259 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,259 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07fe0aab39d9c180ecffd1b70f987a4d, ASSIGN to state=OFFLINE, location=null, table=test, region=07fe0aab39d9c180ecffd1b70f987a4d to restore RIT 2024-11-18T19:12:20,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170265 2024-11-18T19:12:20,265 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,265 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,266 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,267 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,268 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,273 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200273 2024-11-18T19:12:20,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260273 2024-11-18T19:12:20,273 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-18T19:12:20,273 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,274 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,274 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,274 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,274 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,274 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,274 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,275 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,277 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,277 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,279 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,279 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,279 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,279 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN 2024-11-18T19:12:20,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:12:20,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,280 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,280 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,282 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5422fc2846edae2febaf8419c8e7557c, ASSIGN to state=OFFLINE, location=null, table=test, region=5422fc2846edae2febaf8419c8e7557c to restore RIT 2024-11-18T19:12:20,290 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170290 2024-11-18T19:12:20,290 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,290 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,292 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,293 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,293 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,297 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,297 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200297 2024-11-18T19:12:20,297 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260297 2024-11-18T19:12:20,297 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,297 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,297 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,297 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,297 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,298 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,298 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,298 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,302 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,302 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,302 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN 2024-11-18T19:12:20,302 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-18T19:12:20,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,303 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,303 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0653b2f57087b13166483a696eef069d, ASSIGN to state=OFFLINE, location=null, table=test, region=0653b2f57087b13166483a696eef069d to restore RIT 2024-11-18T19:12:20,340 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170340 2024-11-18T19:12:20,340 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,340 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,341 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,342 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,347 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200347 2024-11-18T19:12:20,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260347 2024-11-18T19:12:20,347 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,347 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,347 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,347 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,348 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,348 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN 2024-11-18T19:12:20,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-18T19:12:20,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,353 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,353 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0c0ddbbd34300bf264e72f52607add9, ASSIGN to state=OFFLINE, location=null, table=test, region=a0c0ddbbd34300bf264e72f52607add9 to restore RIT 2024-11-18T19:12:20,373 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170373 2024-11-18T19:12:20,373 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,373 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,375 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,376 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,380 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,381 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200381 2024-11-18T19:12:20,381 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260381 2024-11-18T19:12:20,381 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,381 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,381 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,381 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,381 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,381 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,381 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,381 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,381 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,382 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,384 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,385 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,386 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,387 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,391 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,399 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170399 2024-11-18T19:12:20,400 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,400 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,401 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,403 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,403 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,403 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,406 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200407 2024-11-18T19:12:20,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260407 2024-11-18T19:12:20,407 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,407 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,407 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,407 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,407 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,407 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,407 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.1731957134684, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,411 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-18T19:12:20,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,412 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,412 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,424 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170423 2024-11-18T19:12:20,424 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,424 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,425 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,426 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,427 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,427 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,431 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,431 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200431 2024-11-18T19:12:20,431 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260431 2024-11-18T19:12:20,432 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,432 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,432 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,432 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,432 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,432 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,432 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,432 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,432 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,433 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-18T19:12:20,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,36767,1731957133189 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,438 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:12:20,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,439 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,439 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,449 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170448 2024-11-18T19:12:20,449 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,449 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,450 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,451 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,451 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,451 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,456 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,457 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,457 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200457 2024-11-18T19:12:20,457 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260457 2024-11-18T19:12:20,457 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-18T19:12:20,457 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,457 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,457 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,457 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,457 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,458 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-18T19:12:20,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:12:20,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,464 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,464 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,475 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170475 2024-11-18T19:12:20,475 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,475 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,477 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,483 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200483 2024-11-18T19:12:20,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260483 2024-11-18T19:12:20,483 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-18T19:12:20,483 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,483 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,484 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,484 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,484 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,484 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-18T19:12:20,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,488 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-18T19:12:20,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,489 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,489 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170499 2024-11-18T19:12:20,499 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,499 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,501 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,502 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,502 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,502 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,507 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,507 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200507 2024-11-18T19:12:20,507 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260507 2024-11-18T19:12:20,507 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,507 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,507 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,507 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,507 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,507 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,507 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,507 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,508 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,508 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-18T19:12:20,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 39fff3b0f89c%2C38801%2C1731957133087.meta.1731957135111.meta, worker=39fff3b0f89c,39103,1731957133240 didn't hold the lock before restarting, skip acquiring lock. 2024-11-18T19:12:20,511 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-18T19:12:20,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,512 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,512 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,523 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170523 2024-11-18T19:12:20,524 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,524 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,525 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,528 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-18T19:12:20,528 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,528 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,531 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,531 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200531 2024-11-18T19:12:20,531 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260531 2024-11-18T19:12:20,531 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-18T19:12:20,531 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-18T19:12:20,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,531 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,531 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,531 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-18T19:12:20,532 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-18T19:12:20,532 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-18T19:12:20,532 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:12:20,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T19:12:20,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-18T19:12:20,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:12:20,535 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-18T19:12:20,535 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7468e17fba474acfdc38d26fa77dfaa, REOPEN/MOVE 2024-11-18T19:12:20,535 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true 2024-11-18T19:12:20,535 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-18T19:12:20,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-18T19:12:20,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=65 2024-11-18T19:12:20,536 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=39fff3b0f89c,36767,1731957133189, table=hbase:meta, region=1588230740 2024-11-18T19:12:20,536 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,36767,1731957133189, state=OPEN 2024-11-18T19:12:20,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T19:12:20,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T19:12:20,548 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731957170548 2024-11-18T19:12:20,548 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T19:12:20,548 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T19:12:20,550 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-18T19:12:20,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure e2f1bdafdf390ad35de52e2e494af59c, server=39fff3b0f89c,39103,1731957133240:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 39fff3b0f89c,38801,1731957133087, splitWal=true, meta=true exec-time=2.4680 sec 2024-11-18T19:12:20,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93c84df2c4a5ec3e29ed823586fea37c, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2f1bdafdf390ad35de52e2e494af59c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83c16be18ac836cc02b185da0a033819, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c61a75733b5a3f03b84b6e597527bc5f, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04cf4c4723e9b4f78e59ca75fffd9b53, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2c62ceb958ab1d5bfd5cef6659d0890, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6068e34fbd8b0c94ddb14b0b1840e7aa, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7468e17fba474acfdc38d26fa77dfaa, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=5 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c35b746b07182e41f9cd2f924282671b, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=854fa5bf083055851c2de6bfeca153a3, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65e1250b53aa09c8dc18cb6f054b8ef3, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8a6f4467f3ce2a74edb9e68108f7c16, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5490955be8bafe13acd309a111ae7d57, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9edf883e5dfc34b579ff9c2200b10498, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2df6a61387f05ad72aca10670c6c837a, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=906bcb0fdf3bd087da5078f40abdb11a, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5422fc2846edae2febaf8419c8e7557c, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acb619785982ed899beeeef446d69983, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a1a1d3809d5777735981a521f6cc196d, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07fe0aab39d9c180ecffd1b70f987a4d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5288bd571a792583f30de8b9df67e167, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7059949908a0286d0db393e5fdfea111, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0c0ddbbd34300bf264e72f52607add9, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db50e8b6be6adca837ea205e4b8cb8ea, regionState=OPEN, lastHost=39fff3b0f89c,36767,1731957133189, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0653b2f57087b13166483a696eef069d, regionState=OPENING, lastHost=39fff3b0f89c,38801,1731957133087, regionLocation=39fff3b0f89c,36767,1731957133189, openSeqNum=2 2024-11-18T19:12:20,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e840b4726f8ff2dc5bb7ecc4863af340, regionState=OPEN, lastHost=39fff3b0f89c,39103,1731957133240, regionLocation=39fff3b0f89c,39103,1731957133240, openSeqNum=2 2024-11-18T19:12:20,555 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T19:12:20,555 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731957200555 2024-11-18T19:12:20,555 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731957260555 2024-11-18T19:12:20,555 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-18T19:12:20,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=351 (was 385), OpenFileDescriptor=618 (was 639), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=4562 (was 4608) 2024-11-18T19:12:20,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T19:12:20,571 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:12:20,571 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:12:20,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:12:20,572 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T19:12:20,572 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1074649451, stopped=false 2024-11-18T19:12:20,573 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=39fff3b0f89c,39457,1731957132397 2024-11-18T19:12:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T19:12:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:20,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,581 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:12:20,581 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:12:20,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:20,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:20,582 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,36767,1731957133189' ***** 2024-11-18T19:12:20,582 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T19:12:20,582 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,39103,1731957133240' ***** 2024-11-18T19:12:20,582 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T19:12:20,582 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T19:12:20,582 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T19:12:20,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T19:12:20,582 INFO [RS:1;39fff3b0f89c:36767 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T19:12:20,582 INFO [RS:2;39fff3b0f89c:39103 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T19:12:20,582 INFO [RS:2;39fff3b0f89c:39103 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T19:12:20,582 INFO [RS:1;39fff3b0f89c:36767 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T19:12:20,582 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T19:12:20,582 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T19:12:20,582 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for acb619785982ed899beeeef446d69983 2024-11-18T19:12:20,582 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for c2c62ceb958ab1d5bfd5cef6659d0890 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for db50e8b6be6adca837ea205e4b8cb8ea 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for a0c0ddbbd34300bf264e72f52607add9 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 2df6a61387f05ad72aca10670c6c837a 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for 5422fc2846edae2febaf8419c8e7557c 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 04cf4c4723e9b4f78e59ca75fffd9b53 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for c35b746b07182e41f9cd2f924282671b 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for c61a75733b5a3f03b84b6e597527bc5f 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 5288bd571a792583f30de8b9df67e167 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for 7059949908a0286d0db393e5fdfea111 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 07fe0aab39d9c180ecffd1b70f987a4d 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for a1a1d3809d5777735981a521f6cc196d 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for e2f1bdafdf390ad35de52e2e494af59c 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 83c16be18ac836cc02b185da0a033819 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for 93c84df2c4a5ec3e29ed823586fea37c 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 5490955be8bafe13acd309a111ae7d57 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for 854fa5bf083055851c2de6bfeca153a3 2024-11-18T19:12:20,583 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:20,583 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for 906bcb0fdf3bd087da5078f40abdb11a 2024-11-18T19:12:20,584 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for a8a6f4467f3ce2a74edb9e68108f7c16 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing acb619785982ed899beeeef446d69983, disabling compactions & flushes 2024-11-18T19:12:20,584 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(3091): Received CLOSE for 6068e34fbd8b0c94ddb14b0b1840e7aa 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e840b4726f8ff2dc5bb7ecc4863af340, disabling compactions & flushes 2024-11-18T19:12:20,584 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:20,584 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,584 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;39fff3b0f89c:36767. 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:20,584 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731957135572.acb619785982ed899beeeef446d69983. after waiting 0 ms 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. after waiting 0 ms 2024-11-18T19:12:20,584 INFO [RS:2;39fff3b0f89c:39103 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;39fff3b0f89c:39103. 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:20,584 DEBUG [RS:1;39fff3b0f89c:36767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:12:20,584 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:20,584 DEBUG [RS:1;39fff3b0f89c:36767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,584 DEBUG [RS:2;39fff3b0f89c:39103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:12:20,584 DEBUG [RS:2;39fff3b0f89c:39103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T19:12:20,584 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T19:12:20,584 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T19:12:20,585 DEBUG [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1351): Waiting on 5422fc2846edae2febaf8419c8e7557c, 6068e34fbd8b0c94ddb14b0b1840e7aa, 65e1250b53aa09c8dc18cb6f054b8ef3, 7059949908a0286d0db393e5fdfea111, 854fa5bf083055851c2de6bfeca153a3, 906bcb0fdf3bd087da5078f40abdb11a, 93c84df2c4a5ec3e29ed823586fea37c, a0c0ddbbd34300bf264e72f52607add9, a1a1d3809d5777735981a521f6cc196d, a8a6f4467f3ce2a74edb9e68108f7c16, c61a75733b5a3f03b84b6e597527bc5f, e2f1bdafdf390ad35de52e2e494af59c, e840b4726f8ff2dc5bb7ecc4863af340 2024-11-18T19:12:20,587 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-18T19:12:20,587 DEBUG [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(1351): Waiting on 04cf4c4723e9b4f78e59ca75fffd9b53, 0653b2f57087b13166483a696eef069d, 07fe0aab39d9c180ecffd1b70f987a4d, 1588230740, 2df6a61387f05ad72aca10670c6c837a, 5288bd571a792583f30de8b9df67e167, 5490955be8bafe13acd309a111ae7d57, 83c16be18ac836cc02b185da0a033819, acb619785982ed899beeeef446d69983, c2c62ceb958ab1d5bfd5cef6659d0890, c35b746b07182e41f9cd2f924282671b, db50e8b6be6adca837ea205e4b8cb8ea, e7468e17fba474acfdc38d26fa77dfaa 2024-11-18T19:12:20,587 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T19:12:20,587 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T19:12:20,587 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T19:12:20,587 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T19:12:20,587 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T19:12:20,587 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-18T19:12:20,589 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/acb619785982ed899beeeef446d69983/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,590 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:20,591 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for acb619785982ed899beeeef446d69983: Waiting for close lock at 1731957140583Running coprocessor pre-close hooks at 1731957140583Disabling compacts and flushes for region at 1731957140583Disabling writes for close at 1731957140584 (+1 ms)Writing region close event to WAL at 1731957140585 (+1 ms)Running coprocessor post-close hooks at 1731957140590 (+5 ms)Closed at 1731957140590 2024-11-18T19:12:20,591 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731957135572.acb619785982ed899beeeef446d69983. 2024-11-18T19:12:20,591 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2c62ceb958ab1d5bfd5cef6659d0890, disabling compactions & flushes 2024-11-18T19:12:20,591 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:20,591 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:20,591 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. after waiting 0 ms 2024-11-18T19:12:20,591 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:20,596 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e840b4726f8ff2dc5bb7ecc4863af340/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,596 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c2c62ceb958ab1d5bfd5cef6659d0890/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,597 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:20,597 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:20,597 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e840b4726f8ff2dc5bb7ecc4863af340: Waiting for close lock at 1731957140583Running coprocessor pre-close hooks at 1731957140583Disabling compacts and flushes for region at 1731957140583Disabling writes for close at 1731957140584 (+1 ms)Writing region close event to WAL at 1731957140586 (+2 ms)Running coprocessor post-close hooks at 1731957140597 (+11 ms)Closed at 1731957140597 2024-11-18T19:12:20,597 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2c62ceb958ab1d5bfd5cef6659d0890: Waiting for close lock at 1731957140591Running coprocessor pre-close hooks at 1731957140591Disabling compacts and flushes for region at 1731957140591Disabling writes for close at 1731957140591Writing region close event to WAL at 1731957140592 (+1 ms)Running coprocessor post-close hooks at 1731957140597 (+5 ms)Closed at 1731957140597 2024-11-18T19:12:20,597 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731957135572.e840b4726f8ff2dc5bb7ecc4863af340. 2024-11-18T19:12:20,597 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731957135572.c2c62ceb958ab1d5bfd5cef6659d0890. 2024-11-18T19:12:20,597 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a0c0ddbbd34300bf264e72f52607add9, disabling compactions & flushes 2024-11-18T19:12:20,597 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db50e8b6be6adca837ea205e4b8cb8ea, disabling compactions & flushes 2024-11-18T19:12:20,597 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:20,598 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:20,597 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:20,598 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. after waiting 0 ms 2024-11-18T19:12:20,598 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:20,598 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:20,598 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. after waiting 0 ms 2024-11-18T19:12:20,598 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:20,600 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/info/0b4106ed93234442b8bd2d1bf5914893 is 138, key is test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c./info:regioninfo/1731957139477/Put/seqid=0 2024-11-18T19:12:20,604 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/db50e8b6be6adca837ea205e4b8cb8ea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,604 INFO [regionserver/39fff3b0f89c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T19:12:20,604 INFO [regionserver/39fff3b0f89c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T19:12:20,605 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:20,605 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db50e8b6be6adca837ea205e4b8cb8ea: Waiting for close lock at 1731957140597Running coprocessor pre-close hooks at 1731957140597Disabling compacts and flushes for region at 1731957140597Disabling writes for close at 1731957140598 (+1 ms)Writing region close event to WAL at 1731957140598Running coprocessor post-close hooks at 1731957140604 (+6 ms)Closed at 1731957140605 (+1 ms) 2024-11-18T19:12:20,605 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731957135572.db50e8b6be6adca837ea205e4b8cb8ea. 2024-11-18T19:12:20,605 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2df6a61387f05ad72aca10670c6c837a, disabling compactions & flushes 2024-11-18T19:12:20,605 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:20,605 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:20,605 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. after waiting 0 ms 2024-11-18T19:12:20,605 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:20,612 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a0c0ddbbd34300bf264e72f52607add9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,613 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/2df6a61387f05ad72aca10670c6c837a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,613 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:20,613 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a0c0ddbbd34300bf264e72f52607add9: Waiting for close lock at 1731957140597Running coprocessor pre-close hooks at 1731957140597Disabling compacts and flushes for region at 1731957140597Disabling writes for close at 1731957140598 (+1 ms)Writing region close event to WAL at 1731957140598Running coprocessor post-close hooks at 1731957140613 (+15 ms)Closed at 1731957140613 2024-11-18T19:12:20,613 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731957135572.a0c0ddbbd34300bf264e72f52607add9. 2024-11-18T19:12:20,613 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2df6a61387f05ad72aca10670c6c837a: Waiting for close lock at 1731957140605Running coprocessor pre-close hooks at 1731957140605Disabling compacts and flushes for region at 1731957140605Disabling writes for close at 1731957140605Writing region close event to WAL at 1731957140607 (+2 ms)Running coprocessor post-close hooks at 1731957140613 (+6 ms)Closed at 1731957140613 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5422fc2846edae2febaf8419c8e7557c, disabling compactions & flushes 2024-11-18T19:12:20,614 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731957135572.2df6a61387f05ad72aca10670c6c837a. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. after waiting 0 ms 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 04cf4c4723e9b4f78e59ca75fffd9b53, disabling compactions & flushes 2024-11-18T19:12:20,614 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. after waiting 0 ms 2024-11-18T19:12:20,614 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:20,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741869_1045 (size=8195) 2024-11-18T19:12:20,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741869_1045 (size=8195) 2024-11-18T19:12:20,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741869_1045 (size=8195) 2024-11-18T19:12:20,623 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/04cf4c4723e9b4f78e59ca75fffd9b53/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,623 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/info/0b4106ed93234442b8bd2d1bf5914893 2024-11-18T19:12:20,624 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5422fc2846edae2febaf8419c8e7557c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,624 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:20,624 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 04cf4c4723e9b4f78e59ca75fffd9b53: Waiting for close lock at 1731957140614Running coprocessor pre-close hooks at 1731957140614Disabling compacts and flushes for region at 1731957140614Disabling writes for close at 1731957140614Writing region close event to WAL at 1731957140615 (+1 ms)Running coprocessor post-close hooks at 1731957140624 (+9 ms)Closed at 1731957140624 2024-11-18T19:12:20,624 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731957135572.04cf4c4723e9b4f78e59ca75fffd9b53. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5422fc2846edae2febaf8419c8e7557c: Waiting for close lock at 1731957140614Running coprocessor pre-close hooks at 1731957140614Disabling compacts and flushes for region at 1731957140614Disabling writes for close at 1731957140614Writing region close event to WAL at 1731957140614Running coprocessor post-close hooks at 1731957140624 (+10 ms)Closed at 1731957140624 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731957135572.5422fc2846edae2febaf8419c8e7557c. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c35b746b07182e41f9cd2f924282671b, disabling compactions & flushes 2024-11-18T19:12:20,625 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c61a75733b5a3f03b84b6e597527bc5f, disabling compactions & flushes 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:20,625 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. after waiting 0 ms 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. after waiting 0 ms 2024-11-18T19:12:20,625 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:20,630 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:12:20,630 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:12:20,631 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c61a75733b5a3f03b84b6e597527bc5f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/c35b746b07182e41f9cd2f924282671b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,632 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c61a75733b5a3f03b84b6e597527bc5f: Waiting for close lock at 1731957140625Running coprocessor pre-close hooks at 1731957140625Disabling compacts and flushes for region at 1731957140625Disabling writes for close at 1731957140625Writing region close event to WAL at 1731957140625Running coprocessor post-close hooks at 1731957140632 (+7 ms)Closed at 1731957140632 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731957135572.c61a75733b5a3f03b84b6e597527bc5f. 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7059949908a0286d0db393e5fdfea111, disabling compactions & flushes 2024-11-18T19:12:20,632 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. after waiting 0 ms 2024-11-18T19:12:20,632 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:20,633 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:20,633 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c35b746b07182e41f9cd2f924282671b: Waiting for close lock at 1731957140625Running coprocessor pre-close hooks at 1731957140625Disabling compacts and flushes for region at 1731957140625Disabling writes for close at 1731957140625Writing region close event to WAL at 1731957140625Running coprocessor post-close hooks at 1731957140633 (+8 ms)Closed at 1731957140633 2024-11-18T19:12:20,633 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731957135572.c35b746b07182e41f9cd2f924282671b. 2024-11-18T19:12:20,633 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5288bd571a792583f30de8b9df67e167, disabling compactions & flushes 2024-11-18T19:12:20,633 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:20,634 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:20,634 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/.tmp/info/0b4106ed93234442b8bd2d1bf5914893 as hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/info/0b4106ed93234442b8bd2d1bf5914893 2024-11-18T19:12:20,634 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. after waiting 0 ms 2024-11-18T19:12:20,634 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:20,637 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/7059949908a0286d0db393e5fdfea111/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5288bd571a792583f30de8b9df67e167/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,638 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7059949908a0286d0db393e5fdfea111: Waiting for close lock at 1731957140632Running coprocessor pre-close hooks at 1731957140632Disabling compacts and flushes for region at 1731957140632Disabling writes for close at 1731957140632Writing region close event to WAL at 1731957140633 (+1 ms)Running coprocessor post-close hooks at 1731957140638 (+5 ms)Closed at 1731957140638 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731957135572.7059949908a0286d0db393e5fdfea111. 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a1a1d3809d5777735981a521f6cc196d, disabling compactions & flushes 2024-11-18T19:12:20,638 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:20,638 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. after waiting 0 ms 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5288bd571a792583f30de8b9df67e167: Waiting for close lock at 1731957140633Running coprocessor pre-close hooks at 1731957140633Disabling compacts and flushes for region at 1731957140633Disabling writes for close at 1731957140634 (+1 ms)Writing region close event to WAL at 1731957140634Running coprocessor post-close hooks at 1731957140638 (+4 ms)Closed at 1731957140638 2024-11-18T19:12:20,638 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:20,639 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731957135572.5288bd571a792583f30de8b9df67e167. 2024-11-18T19:12:20,639 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 07fe0aab39d9c180ecffd1b70f987a4d, disabling compactions & flushes 2024-11-18T19:12:20,639 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:20,639 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:20,639 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. after waiting 0 ms 2024-11-18T19:12:20,639 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:20,642 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/info/0b4106ed93234442b8bd2d1bf5914893, entries=24, sequenceid=75, filesize=8.0 K 2024-11-18T19:12:20,644 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=75, compaction requested=false 2024-11-18T19:12:20,644 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a1a1d3809d5777735981a521f6cc196d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,645 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:20,645 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a1a1d3809d5777735981a521f6cc196d: Waiting for close lock at 1731957140638Running coprocessor pre-close hooks at 1731957140638Disabling compacts and flushes for region at 1731957140638Disabling writes for close at 1731957140638Writing region close event to WAL at 1731957140639 (+1 ms)Running coprocessor post-close hooks at 1731957140645 (+6 ms)Closed at 1731957140645 2024-11-18T19:12:20,645 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731957135572.a1a1d3809d5777735981a521f6cc196d. 2024-11-18T19:12:20,645 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/07fe0aab39d9c180ecffd1b70f987a4d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,645 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2f1bdafdf390ad35de52e2e494af59c, disabling compactions & flushes 2024-11-18T19:12:20,645 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:20,646 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:20,646 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. after waiting 0 ms 2024-11-18T19:12:20,646 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:20,646 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:20,646 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 07fe0aab39d9c180ecffd1b70f987a4d: Waiting for close lock at 1731957140639Running coprocessor pre-close hooks at 1731957140639Disabling compacts and flushes for region at 1731957140639Disabling writes for close at 1731957140639Writing region close event to WAL at 1731957140640 (+1 ms)Running coprocessor post-close hooks at 1731957140646 (+6 ms)Closed at 1731957140646 2024-11-18T19:12:20,646 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731957135572.07fe0aab39d9c180ecffd1b70f987a4d. 2024-11-18T19:12:20,646 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83c16be18ac836cc02b185da0a033819, disabling compactions & flushes 2024-11-18T19:12:20,646 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:20,647 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:20,647 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. after waiting 0 ms 2024-11-18T19:12:20,647 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e2f1bdafdf390ad35de52e2e494af59c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T19:12:20,650 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/83c16be18ac836cc02b185da0a033819/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,650 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2f1bdafdf390ad35de52e2e494af59c: Waiting for close lock at 1731957140645Running coprocessor pre-close hooks at 1731957140645Disabling compacts and flushes for region at 1731957140645Disabling writes for close at 1731957140646 (+1 ms)Writing region close event to WAL at 1731957140646Running coprocessor post-close hooks at 1731957140650 (+4 ms)Closed at 1731957140650 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731957140587Running coprocessor pre-close hooks at 1731957140587Disabling compacts and flushes for region at 1731957140587Disabling writes for close at 1731957140587Obtaining lock to block concurrent updates at 1731957140587Preparing flush snapshotting stores in 1588230740 at 1731957140587Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731957140593 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731957140595 (+2 ms)Flushing 1588230740/info: creating writer at 1731957140595Flushing 1588230740/info: appending metadata at 1731957140600 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731957140600Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a8e6589: reopening flushed file at 1731957140633 (+33 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=75, compaction requested=false at 1731957140644 (+11 ms)Writing region close event to WAL at 1731957140646 (+2 ms)Running coprocessor post-close hooks at 1731957140650 (+4 ms)Closed at 1731957140650 2024-11-18T19:12:20,650 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731957135572.e2f1bdafdf390ad35de52e2e494af59c. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 93c84df2c4a5ec3e29ed823586fea37c, disabling compactions & flushes 2024-11-18T19:12:20,651 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. after waiting 0 ms 2024-11-18T19:12:20,651 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83c16be18ac836cc02b185da0a033819: Waiting for close lock at 1731957140646Running coprocessor pre-close hooks at 1731957140646Disabling compacts and flushes for region at 1731957140646Disabling writes for close at 1731957140647 (+1 ms)Writing region close event to WAL at 1731957140647Running coprocessor post-close hooks at 1731957140651 (+4 ms)Closed at 1731957140651 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731957135572.83c16be18ac836cc02b185da0a033819. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5490955be8bafe13acd309a111ae7d57, disabling compactions & flushes 2024-11-18T19:12:20,651 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. after waiting 0 ms 2024-11-18T19:12:20,651 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:20,654 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T19:12:20,659 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/93c84df2c4a5ec3e29ed823586fea37c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,659 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/5490955be8bafe13acd309a111ae7d57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,660 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:20,660 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 93c84df2c4a5ec3e29ed823586fea37c: Waiting for close lock at 1731957140651Running coprocessor pre-close hooks at 1731957140651Disabling compacts and flushes for region at 1731957140651Disabling writes for close at 1731957140651Writing region close event to WAL at 1731957140651Running coprocessor post-close hooks at 1731957140660 (+9 ms)Closed at 1731957140660 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5490955be8bafe13acd309a111ae7d57: Waiting for close lock at 1731957140651Running coprocessor pre-close hooks at 1731957140651Disabling compacts and flushes for region at 1731957140651Disabling writes for close at 1731957140651Writing region close event to WAL at 1731957140652 (+1 ms)Running coprocessor post-close hooks at 1731957140660 (+8 ms)Closed at 1731957140660 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731957135572.5490955be8bafe13acd309a111ae7d57. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731957135572.93c84df2c4a5ec3e29ed823586fea37c. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 854fa5bf083055851c2de6bfeca153a3, disabling compactions & flushes 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7468e17fba474acfdc38d26fa77dfaa, disabling compactions & flushes 2024-11-18T19:12:20,660 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:20,660 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. after waiting 0 ms 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. after waiting 0 ms 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:20,660 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:20,665 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/854fa5bf083055851c2de6bfeca153a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,666 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:20,666 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/e7468e17fba474acfdc38d26fa77dfaa/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-18T19:12:20,666 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 854fa5bf083055851c2de6bfeca153a3: Waiting for close lock at 1731957140660Running coprocessor pre-close hooks at 1731957140660Disabling compacts and flushes for region at 1731957140660Disabling writes for close at 1731957140660Writing region close event to WAL at 1731957140661 (+1 ms)Running coprocessor post-close hooks at 1731957140666 (+5 ms)Closed at 1731957140666 2024-11-18T19:12:20,666 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731957135572.854fa5bf083055851c2de6bfeca153a3. 2024-11-18T19:12:20,666 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 906bcb0fdf3bd087da5078f40abdb11a, disabling compactions & flushes 2024-11-18T19:12:20,666 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:20,667 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:20,667 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. after waiting 0 ms 2024-11-18T19:12:20,667 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:20,667 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:20,667 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7468e17fba474acfdc38d26fa77dfaa: Waiting for close lock at 1731957140660Running coprocessor pre-close hooks at 1731957140660Disabling compacts and flushes for region at 1731957140660Disabling writes for close at 1731957140660Writing region close event to WAL at 1731957140661 (+1 ms)Running coprocessor post-close hooks at 1731957140667 (+6 ms)Closed at 1731957140667 2024-11-18T19:12:20,667 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731957135572.e7468e17fba474acfdc38d26fa77dfaa. 2024-11-18T19:12:20,671 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/906bcb0fdf3bd087da5078f40abdb11a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,672 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:20,672 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 906bcb0fdf3bd087da5078f40abdb11a: Waiting for close lock at 1731957140666Running coprocessor pre-close hooks at 1731957140666Disabling compacts and flushes for region at 1731957140666Disabling writes for close at 1731957140667 (+1 ms)Writing region close event to WAL at 1731957140667Running coprocessor post-close hooks at 1731957140672 (+5 ms)Closed at 1731957140672 2024-11-18T19:12:20,672 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731957135572.906bcb0fdf3bd087da5078f40abdb11a. 2024-11-18T19:12:20,672 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a8a6f4467f3ce2a74edb9e68108f7c16, disabling compactions & flushes 2024-11-18T19:12:20,673 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:20,673 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:20,673 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. after waiting 0 ms 2024-11-18T19:12:20,673 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:20,677 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/a8a6f4467f3ce2a74edb9e68108f7c16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,678 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a8a6f4467f3ce2a74edb9e68108f7c16: Waiting for close lock at 1731957140672Running coprocessor pre-close hooks at 1731957140672Disabling compacts and flushes for region at 1731957140672Disabling writes for close at 1731957140673 (+1 ms)Writing region close event to WAL at 1731957140673Running coprocessor post-close hooks at 1731957140678 (+5 ms)Closed at 1731957140678 2024-11-18T19:12:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731957135572.a8a6f4467f3ce2a74edb9e68108f7c16. 2024-11-18T19:12:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6068e34fbd8b0c94ddb14b0b1840e7aa, disabling compactions & flushes 2024-11-18T19:12:20,678 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. after waiting 0 ms 2024-11-18T19:12:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:20,683 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/6068e34fbd8b0c94ddb14b0b1840e7aa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T19:12:20,683 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:20,683 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6068e34fbd8b0c94ddb14b0b1840e7aa: Waiting for close lock at 1731957140678Running coprocessor pre-close hooks at 1731957140678Disabling compacts and flushes for region at 1731957140678Disabling writes for close at 1731957140678Writing region close event to WAL at 1731957140679 (+1 ms)Running coprocessor post-close hooks at 1731957140683 (+4 ms)Closed at 1731957140683 2024-11-18T19:12:20,683 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731957135572.6068e34fbd8b0c94ddb14b0b1840e7aa. 2024-11-18T19:12:20,717 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-18T19:12:20,785 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,39103,1731957133240; all regions closed. 2024-11-18T19:12:20,787 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,36767,1731957133189; all regions closed. 2024-11-18T19:12:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741834_1010 (size=6795) 2024-11-18T19:12:20,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741834_1010 (size=6795) 2024-11-18T19:12:20,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741834_1010 (size=6795) 2024-11-18T19:12:20,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741865_1041 (size=4204) 2024-11-18T19:12:20,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741865_1041 (size=4204) 2024-11-18T19:12:20,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741865_1041 (size=4204) 2024-11-18T19:12:20,795 DEBUG [RS:2;39fff3b0f89c:39103 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C39103%2C1731957133240:(num 1731957134684) 2024-11-18T19:12:20,796 DEBUG [RS:2;39fff3b0f89c:39103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T19:12:20,796 DEBUG [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs 2024-11-18T19:12:20,796 INFO [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C36767%2C1731957133189.meta:.meta(num 1731957138738) 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T19:12:20,796 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T19:12:20,796 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T19:12:20,797 INFO [RS:2;39fff3b0f89c:39103 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39103 2024-11-18T19:12:20,797 WARN [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,797 WARN [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3., aborting... java.io.IOException: Failed to report opened region to master: test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,798 ERROR [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 39fff3b0f89c,39103,1731957133240: Failed to open region test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,kkk,1731957135572.65e1250b53aa09c8dc18cb6f054b8ef3. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,798 ERROR [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-18T19:12:20,798 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-18T19:12:20,799 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-18T19:12:20,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741833_1009 (size=6268) 2024-11-18T19:12:20,799 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/WALs/39fff3b0f89c,36767,1731957133189/39fff3b0f89c%2C36767%2C1731957133189.1731957134683 not finished, retry = 0 2024-11-18T19:12:20,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741833_1009 (size=6268) 2024-11-18T19:12:20,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741833_1009 (size=6268) 2024-11-18T19:12:20,801 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-18T19:12:20,801 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-18T19:12:20,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T19:12:20,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,39103,1731957133240 2024-11-18T19:12:20,806 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T19:12:20,806 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,39103,1731957133240] 2024-11-18T19:12:20,807 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 496946048 }, "NonHeapMemoryUsage": { "committed": 163381248, "init": 7667712, "max": -1, "used": 160927216 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 41976, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 138, "ProcessCallTime_min": 0, "ProcessCallTime_max": 32, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 16, "ProcessCallTime_75th_percentile": 24, "ProcessCallTime_90th_percentile": 28, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 31, "ProcessCallTime_99th_percentile": 31, "ProcessCallTime_99.9th_percentile": 31, "ProcessCallTime_TimeRangeCount_0-1": 138, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 138, "QueueCallTime_min": 0, "QueueCallTime_max": 8, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 4, "QueueCallTime_75th_percentile": 6, "QueueCallTime_90th_percentile": 7, "QueueCallTime_95th_percentile": 7, "QueueCallTime_98th_percentile": 7, "QueueCallTime_99th_percentile": 7, "QueueCallTime_99.9th_percentile": 7, "QueueCallTime_TimeRangeCount_0-1": 138, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 138, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 8, "TotalCallTime_median": 16, "TotalCallTime_75th_percentile": 24, "TotalCallTime_90th_percentile": 28, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 31, "TotalCallTime_99th_percentile": 31, "TotalCallTime_99.9th_percentile": 31, "TotalCallTime_TimeRangeCount_0-1": 138, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 138, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 4304, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 138, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 138, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 276, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 138, "sentBytes": 459952 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49195", "tag.serverName": "39fff3b0f89c,38801,1731957133087", "tag.clusterId": "36bffa65-ad57-45e9-886f-853e0e6c4ad1", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731957133087, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 39673, "localBytesRead": 39673, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 40861, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 11, "ScanTime_mean": 1, "ScanTime_25th_percentile": 2, "ScanTime_median": 5, "ScanTime_75th_percentile": 8, "ScanTime_90th_percentile": 9, "ScanTime_95th_percentile": 10, "ScanTime_98th_percentile": 10, "ScanTime_99th_percentile": 10, "ScanTime_99.9th_percentile": 10, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 8, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 6, "Put_90th_percentile": 7, "Put_95th_percentile": 7, "Put_98th_percentile": 7, "Put_99th_percentile": 7, "Put_99.9th_percentile": 7, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 10, "PutBatch_max": 10, "PutBatch_mean": 10, "PutBatch_25th_percentile": 10, "PutBatch_median": 10, "PutBatch_75th_percentile": 10, "PutBatch_90th_percentile": 10, "PutBatch_95th_percentile": 10, "PutBatch_98th_percentile": 10, "PutBatch_99th_percentile": 10, "PutBatch_99.9th_percentile": 10, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-18T19:12:20,822 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,39103,1731957133240 already deleted, retry=false 2024-11-18T19:12:20,822 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39fff3b0f89c,39103,1731957133240 expired; onlineServers=1 2024-11-18T19:12:20,859 WARN [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,yyy,1731957135572.0653b2f57087b13166483a696eef069d., aborting... java.io.IOException: Failed to report opened region to master: test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,859 ERROR [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 39fff3b0f89c,36767,1731957133189: Failed to open region test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,yyy,1731957135572.0653b2f57087b13166483a696eef069d. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,859 ERROR [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-18T19:12:20,860 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-18T19:12:20,861 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-18T19:12:20,862 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-18T19:12:20,862 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-18T19:12:20,866 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 499043200 }, "NonHeapMemoryUsage": { "committed": 163643392, "init": 7667712, "max": -1, "used": 161150472 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 41976, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 138, "ProcessCallTime_min": 0, "ProcessCallTime_max": 32, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 16, "ProcessCallTime_75th_percentile": 24, "ProcessCallTime_90th_percentile": 28, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 31, "ProcessCallTime_99th_percentile": 31, "ProcessCallTime_99.9th_percentile": 31, "ProcessCallTime_TimeRangeCount_0-1": 138, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 138, "QueueCallTime_min": 0, "QueueCallTime_max": 8, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 4, "QueueCallTime_75th_percentile": 6, "QueueCallTime_90th_percentile": 7, "QueueCallTime_95th_percentile": 7, "QueueCallTime_98th_percentile": 7, "QueueCallTime_99th_percentile": 7, "QueueCallTime_99.9th_percentile": 7, "QueueCallTime_TimeRangeCount_0-1": 138, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 138, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 8, "TotalCallTime_median": 16, "TotalCallTime_75th_percentile": 24, "TotalCallTime_90th_percentile": 28, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 31, "TotalCallTime_99th_percentile": 31, "TotalCallTime_99.9th_percentile": 31, "TotalCallTime_TimeRangeCount_0-1": 138, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 138, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 4304, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 138, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 138, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 276, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 138, "sentBytes": 459952 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49195", "tag.serverName": "39fff3b0f89c,38801,1731957133087", "tag.clusterId": "36bffa65-ad57-45e9-886f-853e0e6c4ad1", "tag.Context": "regionserver", "tag.Hostname": "39fff3b0f89c", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731957133087, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 39673, "localBytesRead": 39673, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 40861, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 11, "ScanTime_mean": 1, "ScanTime_25th_percentile": 2, "ScanTime_median": 5, "ScanTime_75th_percentile": 8, "ScanTime_90th_percentile": 9, "ScanTime_95th_percentile": 10, "ScanTime_98th_percentile": 10, "ScanTime_99th_percentile": 10, "ScanTime_99.9th_percentile": 10, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 8, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 6, "Put_90th_percentile": 7, "Put_95th_percentile": 7, "Put_98th_percentile": 7, "Put_99th_percentile": 7, "Put_99.9th_percentile": 7, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 10, "PutBatch_max": 10, "PutBatch_mean": 10, "PutBatch_25th_percentile": 10, "PutBatch_median": 10, "PutBatch_75th_percentile": 10, "PutBatch_90th_percentile": 10, "PutBatch_95th_percentile": 10, "PutBatch_98th_percentile": 10, "PutBatch_99th_percentile": 10, "PutBatch_99.9th_percentile": 10, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-18T19:12:20,868 WARN [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=39fff3b0f89c:39457 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=39fff3b0f89c:39457 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more 2024-11-18T19:12:20,870 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:20,870 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 9edf883e5dfc34b579ff9c2200b10498, NAME => 'test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-18T19:12:20,870 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,870 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:12:20,870 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,870 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,872 INFO [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,873 INFO [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9edf883e5dfc34b579ff9c2200b10498 columnFamilyName family 2024-11-18T19:12:20,873 DEBUG [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:12:20,874 INFO [StoreOpener-9edf883e5dfc34b579ff9c2200b10498-1 {}] regionserver.HStore(327): Store=9edf883e5dfc34b579ff9c2200b10498/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:12:20,874 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,875 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,877 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/data/default/test/9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,877 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,877 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,879 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,881 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 9edf883e5dfc34b579ff9c2200b10498; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63989322, jitterRate=-0.046484798192977905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:12:20,881 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9edf883e5dfc34b579ff9c2200b10498 2024-11-18T19:12:20,881 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 9edf883e5dfc34b579ff9c2200b10498: Running coprocessor pre-open hook at 1731957140871Writing region info on filesystem at 1731957140871Initializing all the Stores at 1731957140872 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731957140872Cleaning up temporary data from old regions at 1731957140877 (+5 ms)Running coprocessor post-open hooks at 1731957140881 (+4 ms)Region opened successfully at 1731957140881 2024-11-18T19:12:20,881 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1722): Closing 9edf883e5dfc34b579ff9c2200b10498, disabling compactions & flushes 2024-11-18T19:12:20,881 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1755): Closing region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:20,881 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:20,881 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. after waiting 0 ms 2024-11-18T19:12:20,881 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:20,882 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1973): Closed test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. 2024-11-18T19:12:20,882 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1676): Region close journal for 9edf883e5dfc34b579ff9c2200b10498: Waiting for close lock at 1731957140881Running coprocessor pre-close hooks at 1731957140881Disabling compacts and flushes for region at 1731957140881Disabling writes for close at 1731957140881Writing region close event to WAL at 1731957140882 (+1 ms)Running coprocessor post-close hooks at 1731957140882Closed at 1731957140882 2024-11-18T19:12:20,882 WARN [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(84): Failed to open region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498., will report to master java.io.IOException: Cannot append; log is closed, regionName = test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1393) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.append(AbstractFSWAL.java:1920) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$appendMarker$14(AbstractFSWAL.java:1455) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendMarker(AbstractFSWAL.java:1455) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullMarkerAppendTransaction(WALUtil.java:168) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeMarker(WALUtil.java:148) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeRegionEventMarker(WALUtil.java:112) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.writeRegionOpenMarker(HRegion.java:1269) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7808) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,882 WARN [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498., aborting... java.io.IOException: Failed to report failed open to master: test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:12:20,883 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,nnn,1731957135572.9edf883e5dfc34b579ff9c2200b10498. and can not recover 2024-11-18T19:12:20,902 DEBUG [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/oldWALs 2024-11-18T19:12:20,902 INFO [RS:1;39fff3b0f89c:36767 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C36767%2C1731957133189:(num 1731957134683) 2024-11-18T19:12:20,902 DEBUG [RS:1;39fff3b0f89c:36767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:12:20,902 INFO [RS:1;39fff3b0f89c:36767 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:12:20,902 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T19:12:20,903 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T19:12:20,903 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T19:12:20,903 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T19:12:20,903 INFO [RS:1;39fff3b0f89c:36767 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36767 2024-11-18T19:12:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T19:12:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36767-0x10150a63b420002, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,36767,1731957133189 2024-11-18T19:12:20,914 INFO [RS:1;39fff3b0f89c:36767 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T19:12:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:12:20,914 INFO [RS:2;39fff3b0f89c:39103 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T19:12:20,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39103-0x10150a63b420003, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:12:20,914 INFO [RS:2;39fff3b0f89c:39103 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,39103,1731957133240; zookeeper connection closed. 2024-11-18T19:12:20,915 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a951bb4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a951bb4 2024-11-18T19:12:20,922 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,36767,1731957133189] 2024-11-18T19:12:20,930 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,36767,1731957133189 already deleted, retry=false 2024-11-18T19:12:20,930 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39fff3b0f89c,36767,1731957133189 expired; onlineServers=0 2024-11-18T19:12:20,931 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '39fff3b0f89c,39457,1731957132397' ***** 2024-11-18T19:12:20,931 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T19:12:20,931 INFO [M:0;39fff3b0f89c:39457 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T19:12:20,931 INFO [M:0;39fff3b0f89c:39457 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T19:12:20,931 DEBUG [M:0;39fff3b0f89c:39457 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T19:12:20,931 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T19:12:20,931 DEBUG [M:0;39fff3b0f89c:39457 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T19:12:20,931 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731957134289 {}] cleaner.HFileCleaner(306): Exit Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731957134289,5,FailOnTimeoutGroup] 2024-11-18T19:12:20,931 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731957134290 {}] cleaner.HFileCleaner(306): Exit Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731957134290,5,FailOnTimeoutGroup] 2024-11-18T19:12:20,931 INFO [M:0;39fff3b0f89c:39457 {}] hbase.ChoreService(370): Chore service for: master/39fff3b0f89c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T19:12:20,931 INFO [M:0;39fff3b0f89c:39457 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T19:12:20,931 DEBUG [M:0;39fff3b0f89c:39457 {}] master.HMaster(1795): Stopping service threads 2024-11-18T19:12:20,932 INFO [M:0;39fff3b0f89c:39457 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T19:12:20,932 INFO [M:0;39fff3b0f89c:39457 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:12:20,932 INFO [M:0;39fff3b0f89c:39457 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:12:20,932 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T19:12:20,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T19:12:20,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T19:12:20,939 DEBUG [M:0;39fff3b0f89c:39457 {}] zookeeper.ZKUtil(347): master:39457-0x10150a63b420000, quorum=127.0.0.1:49195, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T19:12:20,939 WARN [M:0;39fff3b0f89c:39457 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T19:12:20,940 INFO [M:0;39fff3b0f89c:39457 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/.lastflushedseqids 2024-11-18T19:12:20,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741870_1046 (size=1) 2024-11-18T19:12:20,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741870_1046 (size=1) 2024-11-18T19:12:20,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741870_1046 (size=1) 2024-11-18T19:12:20,952 INFO [M:0;39fff3b0f89c:39457 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:12:20,952 INFO [M:0;39fff3b0f89c:39457 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T19:12:20,952 DEBUG [M:0;39fff3b0f89c:39457 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T19:12:20,952 INFO [M:0;39fff3b0f89c:39457 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:20,952 DEBUG [M:0;39fff3b0f89c:39457 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:20,952 DEBUG [M:0;39fff3b0f89c:39457 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T19:12:20,952 DEBUG [M:0;39fff3b0f89c:39457 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:12:20,953 INFO [M:0;39fff3b0f89c:39457 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.85 KB heapSize=315.83 KB 2024-11-18T19:12:20,970 DEBUG [M:0;39fff3b0f89c:39457 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/641b2860bbd643ecb1431539630f8d09 is 82, key is hbase:meta,,1/info:regioninfo/1731957139059/Put/seqid=0 2024-11-18T19:12:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35199 is added to blk_1073741871_1047 (size=6071) 2024-11-18T19:12:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35133 is added to blk_1073741871_1047 (size=6071) 2024-11-18T19:12:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41665 is added to blk_1073741871_1047 (size=6071) 2024-11-18T19:12:20,979 INFO [M:0;39fff3b0f89c:39457 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/641b2860bbd643ecb1431539630f8d09 2024-11-18T19:12:21,004 DEBUG [M:0;39fff3b0f89c:39457 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42667/user/jenkins/test-data/3711e60c-d988-d20c-632b-c450cc965b53/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6298fe07701d4d54b14a567474420d64 is 2990, key is \x00\x00\x00\x00\x00\x00\x00